【Python】爬虫(Xpath):批量爬取织梦免费模版

织梦站内有很多的一些免费模版给非会员用户下载, 此次爬取免费模版并非实际意义的下载, 因为站内并没有直接下载,都是使用百度网盘分享链接的方式下载的,所以本次爬取到的是各个模板相对应的分享链接和提取码

import requests
import os
from lxml import etree


def demo(page_need, headers):
    desktop = os.path.join(os.path.expanduser("~"), 'Desktop')
    fp = open(desktop + '/织梦免费模版.txt', 'w')
    for i in range(int(page_need)):
        url = "http://www.adminbuy.cn/dedecms/list_2_{}.html".format(page_need)
        list_page_text = requests.get(url=url, headers=headers).text
        list_page_tree = etree.HTML(list_page_text)
        article_url = list_page_tree.xpath('/html/body/div[7]/div/div[5]/div/div/a/@href')
        for url in article_url:
            article_page_text = requests.get(url=url, headers=headers).text
            article_page_tree = etree.HTML(article_page_text)
            name = article_page_tree.xpath('//*[@id="goodsInfo"]/div[2]/h2/text()')[0]
            name = name.encode('ISO-8859-1').decode('gbk')
            url_text = article_page_tree.xpath('//*[@id="goodsInfo"]/div[2]/div[2]/li/div/span/a/@href')[0]
            pw = article_page_tree.xpath('//*[@id="goodsInfo"]/div[2]/div[2]/li/div/span/i/text()')[0]
            pw = pw.encode('ISO-8859-1').decode('gbk')
            fp.write('标题: {}\n提取链接: {}  {}\n\n'.format(name, url_text, pw))



url = 'http://www.adminbuy.cn/dedecms/list_2_1.html'
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3775.400 QQBrowser/10.6.4208.400",
}
list_page_text = requests.get(url=url, headers=headers).text
list_page_tree = etree.HTML(list_page_text)
page_limit = list_page_tree.xpath('/html/body/div[7]/div/div[5]/div[17]/li[19]/span/strong[1]/text()')[0]
print("页数上限为{}".format(page_limit))
page_need = input("请输入需要的页数: ")

if page_need.isdigit() and 1 <= int(page_need) <= int(page_limit):
    demo(page_need, headers)