以李清照为例,爬取结果展示:
代码实现:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time : 2020/11/30 19:59
# @Author : huni
# @File : 古诗词作者作品.py
# @Software: PyCharm
import requests
from lxml import etree
if __name__ == '__main__':
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.183 Safari/537.36'
}
for i in range(1,12):
url = f'https://so.gushiwen.org/shiwen/default_2A9cb3b7c0e4a0A{i}.aspx'
page_text = requests.get(url=url,headers=headers).text
e = etree.HTML(page_text)
div_list = e.xpath('//div[@class="left"]/div[@class="sons"]')
for div in div_list:
title = div.xpath('./div[1]/p[1]//text()')[0]
author = ''.join(div.xpath('./div[1]/p[2]//text()'))
contents = div.xpath('./div/div[2]//text()')
with open('./李清照.txt','a',encoding='utf-8') as fp:
fp.write(title + '\n' + author)
for content in contents:
with open('./李清照.txt', 'a',encoding='utf-8') as fp:
fp.write(content)
with open('./李清照.txt', 'a', encoding='utf-8') as fp:
fp.write('\n')
print('写入完成!')