百度爬取100张狗狗图片

因为百度上狗狗图片是动态加载的,用requests爬取到的图片只有30张,所以去我选择用selenium.为了方便分析网页代码,我习惯将代码写入文件,方便查看异常,另外还可以避免重复爬取被拉黑.代码如下:

import requests
from lxml import etree
from selenium import webdriver

url = "https://image.baidu.com/search/index?tn=baiduimage&word=100%D5%C5%B9%B7%CD%BC%C6%AC"
driver = webdriver.Chrome(r"C:\Users\ling li\AppData\Local\Google\Chrome\Application\chromedriver.exe")
driver.get(url)
text = driver.execute_script("return document.documentElement.outerHTML")
with open("狗图片网页代码2","w",encoding="utf-8") as f:
    f.write(text)
# with open("狗图片网页代码2",encoding="utf-8") as f:
#     text = f.read()
tree = etree.HTML(text)

url_list = []
i = 0
for each in tree.xpath('//div[@class="imgbox"]//a//img'):
    i += 1
    if i<101:
        url_list.append((each.xpath("./@data-imgurl"))[0])
        img_url = each.xpath("./@data-imgurl")[0]
        response = requests.get(img_url)
        with open("dog/狗狗%d.jpg"%i,"wb") as f:
            f.write(response.content)
print(len(url_list))
print(url_list)