因为百度上狗狗图片是动态加载的,用requests爬取到的图片只有30张,所以去我选择用selenium.为了方便分析网页代码,我习惯将代码写入文件,方便查看异常,另外还可以避免重复爬取被拉黑.代码如下:
import requests
from lxml import etree
from selenium import webdriver
url = "https://image.baidu.com/search/index?tn=baiduimage&word=100%D5%C5%B9%B7%CD%BC%C6%AC"
driver = webdriver.Chrome(r"C:\Users\ling li\AppData\Local\Google\Chrome\Application\chromedriver.exe")
driver.get(url)
text = driver.execute_script("return document.documentElement.outerHTML")
with open("狗图片网页代码2","w",encoding="utf-8") as f:
f.write(text)
# with open("狗图片网页代码2",encoding="utf-8") as f:
# text = f.read()
tree = etree.HTML(text)
url_list = []
i = 0
for each in tree.xpath('//div[@class="imgbox"]//a//img'):
i += 1
if i<101:
url_list.append((each.xpath("./@data-imgurl"))[0])
img_url = each.xpath("./@data-imgurl")[0]
response = requests.get(img_url)
with open("dog/狗狗%d.jpg"%i,"wb") as f:
f.write(response.content)
print(len(url_list))
print(url_list)