Python运用Xpath爬取故宫壁纸

Python运用Xpath爬取故宫壁纸

网址： link.

1.关于获取"User-Agent"

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) '
                  'Chrome/86.0.4240.193 '
                  'Safari/537.36 '
}

2.关于获取图片的名称

//*[@id="lights"]/div[2]/div/h3/text()

3.关于获取图片的地址

3.1 在预览壁纸的网页里获取图片的位置信息

//div[@class = 'pic']//@href

3.2 获取图片下载地址信息

//img[@style ='visibility: visible;width: 100%;']/@src

爬取图片下载的网址

4.获取下一页的网址信息

5.结果展示

6.代码

import requests
from lxml import etree
# 读写文件
import os
# 进程
from multiprocessing import Process


headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) '
                  'Chrome/86.0.4240.193 '
                  'Safari/537.36 '
}



def Down_Images(url, image_title):
    response_images = requests.get(url, headers=headers).text
    # print(response_images)
    html1 = etree.HTML(response_images)
    image_data = html1.xpath("//img[@style ='visibility: visible;width: 100%;']/@src")
    # print(image_data[0])
    image_url = requests.get(image_data[0], headers=headers)
    p0 = image_title
    p2 = image_data
    # print(p1)
    # print(p2[0])
    url = p2[0]
    # 下载png图片放到./壁纸/文件夹里面
    data = requests.get(url, headers=headers).content
    print(p0 + " " + url)
    p = str(p0)
    # 将文件夹路径传入
    img_path = "./壁纸/"
    if not os.path.exists(img_path):
        os.makedirs('./壁纸/')
    else:
        if '\t' not in p:
            f = open(img_path + p + ".png", "wb")
            f.write(data)
            f.close()
        else:
            f = open(img_path + p.replace('\t', "One") + ".png", "wb")
            f.write(data)
            f.close()



def Get_URL(page):
    url = 'https://www.dpm.org.cn/lights/royal/p/' + str(page) + '.html'
    response_data = requests.get(url, headers=headers)
    html = etree.HTML(response_data.content.decode('utf-8', 'ignore'))
    images_coup = html.xpath("//div[@class = 'pic']//@href")
    image_title = html.xpath('//*[@id="lights"]/div[2]/div/h3/text()')
    n = 0
    for i in images_coup:
        url_image = "https://www.dpm.org.cn" + i
        Down_Images(url_image, image_title[n])
        n += 1


if __name__ == '__main__':
    # 创建一个进程
    # Process(指向->函数,函数的参数)
    for i in range(1, 10):
        # 后面这个参数是一个元祖
        p1 = Process(target=Get_URL, args=(i,))

        # 运行进程
        p1.start()

        # 等待进程结束
        p1.join()