爬虫学习之xpath:使用requests、xpath、html.etree、图片懒加载、转换编码等，爬取多页图片并下载

# -*- coding: utf-8 -*-
import os,requests
from lxml import html
headers={
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36'
}
fileName = 'TestFiles'
if not os.path.exists(fileName):
    os.mkdir(fileName)
url = 'http://sc.chinaz.com/tupian/index_%d.html'
for page in range(1,2):
    if page == 1:
        img_url = 'http://sc.chinaz.com/tupian/'
    else:
        img_url = format(url%page)
    response = requests.get(url=img_url,headers=headers)
    response.encoding = 'utf-8'
    page_text = response.text
    tree = html.etree.HTML(page_text)
    a_list = tree.xpath('//div/a/img')
    for a in a_list:
        img_src = a.xpath('./@src2')[0][:-6] + '.jpg'   #动态懒加载的网站，使用抓包工具查找真正的图片地址并留意图片_s
        img_name = a.xpath('./@alt')[0] + '.jpg'
        img_data = requests.get(url=img_src,headers=headers).content
        img_path = fileName + '/' + img_name
        with open(img_path,'wb') as fp:
            fp.write(img_data)
        print(img_name +':'+'保存成功')