Python-爬虫（爬虫练习 爬取古诗文网五言绝句）

目标网站

采用的数据解析方式：xpath、bs4、re正则

获取网站中所有的五言绝句诗词链接

from bs4 import BeautifulSoup
import re


# 获取五言绝句代码链接，以列表的形式返回
def _getLink(html):
    html_data = BeautifulSoup(html, 'lxml')
    five_html = str(html_data.find_all('div', class_="typecont")[0])

    ret = re.findall('<span><a href="(.*?)" target="_blank">', five_html)
    return ret

获取对应古诗页面中，古诗的题目和内容

from lxml import etree


# 合并列表中所有字符串
def GetMsg(iterable):
    ret = ""
    for i in iterable:
        ret += i
    return ret


# 获取古诗题目，作者
def _getPoetryTitle(data):
    html = etree.HTML(data)
    title = GetMsg(html.xpath('//*[@id="sonsyuanwen"]/div[1]/h1/text()'))
    msg = GetMsg(html.xpath('//*[@id="sonsyuanwen"]/div[1]/p/a/text()'))
    return title + " " + msg


# 获取古诗正文
def _getPoetryText(data):
    html = etree.HTML(data)
    # //*[@id = "sonsyuanwen"]/div[1]/div[2]
    msg = GetMsg(html.xpath('//*[@id ="sonsyuanwen"]/div[1]/div[2]/text()'))
    return msg

主函数爬取过程

import requests

from getLink import _getLink
from getPoetry import _getPoetryText, _getPoetryTitle

url_root = "https://so.gushiwen.cn"

main_page = "/gushi/tangshi.aspx"

headers = {
    'user-agent':
        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 '
        'Safari/537.36 '
}

if __name__ == '__main__':
    response = requests.get(url_root + main_page, headers=headers)

    file = open("五言绝句.txt", 'w', encoding='utf-8')

    if response.status_code != 200:
        print('获取主页面失败，程序退出')
        exit(-1)
    html = response.text

    # _getLink获取所有要请求的链接
    link = _getLink(html)

    # 请求每一个链接
    for _url in link:
        _response = requests.get(url_root + _url, headers=headers)
        if response.status_code != 200:
            print('获取子页面失败，程序退出')
            exit(-1)
        data = _response.text
        if file is None:
            exit(-1)
        print(f'{_getPoetryTitle(data)}爬取开始')
        file.write(f"                              {_getPoetryTitle(data)}\n")
        # 写入取诗歌内容
        file.write(_getPoetryText(data))
        file.write("====================================================\n")
        print(f'{_getPoetryTitle(data)}爬取结束')

    # 写入完成后关闭文件
    file.close()