采用的数据解析方式:xpath、bs4、re正则
获取网站中所有的五言绝句诗词链接
from bs4 import BeautifulSoup
import re
# 获取五言绝句代码链接,以列表的形式返回
def _getLink(html):
html_data = BeautifulSoup(html, 'lxml')
five_html = str(html_data.find_all('div', class_="typecont")[0])
ret = re.findall('<span><a href="(.*?)" target="_blank">', five_html)
return ret
获取对应古诗页面中,古诗的题目和内容
from lxml import etree
# 合并列表中所有字符串
def GetMsg(iterable):
ret = ""
for i in iterable:
ret += i
return ret
# 获取古诗题目,作者
def _getPoetryTitle(data):
html = etree.HTML(data)
title = GetMsg(html.xpath('//*[@id="sonsyuanwen"]/div[1]/h1/text()'))
msg = GetMsg(html.xpath('//*[@id="sonsyuanwen"]/div[1]/p/a/text()'))
return title + " " + msg
# 获取古诗正文
def _getPoetryText(data):
html = etree.HTML(data)
# //*[@id = "sonsyuanwen"]/div[1]/div[2]
msg = GetMsg(html.xpath('//*[@id ="sonsyuanwen"]/div[1]/div[2]/text()'))
return msg
主函数爬取过程
import requests
from getLink import _getLink
from getPoetry import _getPoetryText, _getPoetryTitle
url_root = "https://so.gushiwen.cn"
main_page = "/gushi/tangshi.aspx"
headers = {
'user-agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 '
'Safari/537.36 '
}
if __name__ == '__main__':
response = requests.get(url_root + main_page, headers=headers)
file = open("五言绝句.txt", 'w', encoding='utf-8')
if response.status_code != 200:
print('获取主页面失败,程序退出')
exit(-1)
html = response.text
# _getLink获取所有要请求的链接
link = _getLink(html)
# 请求每一个链接
for _url in link:
_response = requests.get(url_root + _url, headers=headers)
if response.status_code != 200:
print('获取子页面失败,程序退出')
exit(-1)
data = _response.text
if file is None:
exit(-1)
print(f'{_getPoetryTitle(data)}爬取开始')
file.write(f" {_getPoetryTitle(data)}\n")
# 写入取诗歌内容
file.write(_getPoetryText(data))
file.write("====================================================\n")
print(f'{_getPoetryTitle(data)}爬取结束')
# 写入完成后关闭文件
file.close()