xpath解析(2)

网址：https://www.58pic.com/piccate/10-0-0-p12.html

# (1) 请求对象的定制
# （2）获取网页的源码
# （3）下载


# 需求 下载的前十页的图片
import urllib.request
from lxml import etree

def create_request(page):
    # https://www.58pic.com/piccate/10-0-0.html
    # https://www.58pic.com/piccate/10-0-0-p04.html
    # https://www.58pic.com/piccate/10-0-0-p06.html
    if (page == 1):
        url = 'https://www.58pic.com/piccate/10-0-0.html'
    else:
        url = 'https://www.58pic.com/piccate/10-0-0-p0'+str(page) + '.html'

    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36',
    }
    request = urllib.request.Request(url=url, headers=headers)
    return request
    pass

def get_content(request):
    response = urllib.request.urlopen(request)
    # content = response.read().decode('ISO-8859-1')
    content = response.read().decode('GB2312')
    return content


def down_load(content):
    tree = etree.HTML(content)
    name_list = tree.xpath('//div[@class = "qt-card-box qtas-pic-card is-complex"]//div[@class = "card-text-box"]//p/text()')
    src_list = tree.xpath('//div[@class = "qt-card-box qtas-pic-card is-complex"]//div[@class = "card-img-box"]/img/@data-original')


    for i in range(len(name_list)):
        name = name_list[i]
        src = src_list[i]
        url = 'https:' + src
        urllib.request.urlretrieve(url = url, filename='./backImg/' + name + '.jpg')


if __name__ == '__main__':
    start_page = int(input("请输入起始页码："))
    end_page = int(input("请输入结束页码："))

    for page in range(start_page, end_page + 1):
        # 请求对象定制
        request = create_request(page)
        # 获取网页源码
        content = get_content(request)
        # 下载图片
        down_load(content)

难点：

• 懒加载模式：
  –
  这里要用data-origin 而非src
• 解码方式：utf-8不能解码（复杂汉字的存在），解码方式应该使用GB2312