python爬虫lxml解析爬取诗词名句

原创：仅用于学习Python爬虫，请勿商业或恶意爬取数据
文件夹和文件都是程序创建，我只爬了这些数据用于测试
仅用了两个for循环，并没有搞的太难（函数），适合新手操练，有大量注释易于理解

from lxml import etree
import requests
import os
from multiprocessing import Pool
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36",
    'Referer':'http://www.shicimingju.com/'
}
# 设置url
url = 'http://www.shicimingju.com/book/'
# 请求数据

response = requests.get(url=url,headers=headers)
# 获取页面数据
page_text = response.text
# 转换成可解析对象
etrees = etree.HTML(page_text)
# 解析数据
h2_list = etrees.xpath('//div[@class="bookmark-list"]/ul/li/h2')

# 获取某本书的目录
for i in h2_list:
    # 获取链接到目录url
    src_list = i.xpath("./a/@href")[0]
    # print(src_list)     #/book/shuotang.html
    src_url = url + src_list
    # 获取链接到目录 目录名
    a_name = i.xpath("./a/text()")[0]
    # print(a_name)       #说唐
    # 根据目录链接进入某本书的目录页面
    response = requests.get(url=src_url,headers=headers)
    # 获取页面数据
    page_text = response.text
    etrees = etree.HTML(page_text)
    # 解析页面 准备找url和章节名称
    li_list = etrees.xpath('//div[@class="book-mulu"]/ul/li')
    file_path = "./史书典籍/%s"%a_name + "/"
    if not os.path.exists(file_path):
        os.mkdir(file_path)
    for j in li_list:
        # 单篇小说的某个章节名字
        a_text_singles = j.xpath("./a/text()")[0]
        # 单篇小说的某个章节的url
        a_url_singles = j.xpath("./a/@href")[0]
        a_url = url + a_url_singles
        response = requests.get(url=a_url, headers=headers)
        # 获取页面数据
        page_text = response.text
        # 转成可解析的对象
        etrees = etree.HTML(page_text)
        content = etrees.xpath('//div[@class="chapter_content"]/p/text()')
        # 获取到的列表转为字符串
        content = "".join(content)
        file_name = file_path + a_text_singles +".txt"
        with open(file_name,"w",encoding="utf-8") as f:
            f.write(content)
        print("%s,%s下载完成"%(a_name,a_text_singles))
    print("%s下载完成"%a_name)