python 爬虫 古诗词网按照作者爬取所有作品

以李清照为例，爬取结果展示：

代码实现：

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2020/11/30 19:59
# @Author  : huni
# @File    : 古诗词作者作品.py
# @Software: PyCharm

import requests
from lxml import etree

if __name__ == '__main__':
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.183 Safari/537.36'
    }
    for i in range(1,12):
        url = f'https://so.gushiwen.org/shiwen/default_2A9cb3b7c0e4a0A{i}.aspx'

        page_text = requests.get(url=url,headers=headers).text

        e = etree.HTML(page_text)

        div_list = e.xpath('//div[@class="left"]/div[@class="sons"]')

        for div in div_list:
            title = div.xpath('./div[1]/p[1]//text()')[0]
            author = ''.join(div.xpath('./div[1]/p[2]//text()'))
            contents = div.xpath('./div/div[2]//text()')

            with open('./李清照.txt','a',encoding='utf-8') as fp:
                fp.write(title + '\n' + author)
            for content in contents:
                with open('./李清照.txt', 'a',encoding='utf-8') as fp:
                    fp.write(content)
            with open('./李清照.txt', 'a', encoding='utf-8') as fp:
                fp.write('\n')
    print('写入完成！')