Python爬取古诗词

一、需求

爬取网址：https://www.gushiwen.org/

需求：

（1）获取侧边栏【类型】信息；
（2）获取每个类型中古诗文详情页信息；
（3）提取详情页数据：古诗文名、作者、朝代、类型、内容、译文及注释；
（4）将数据保存到 csv 文件；

二、代码实现

import requests
import csv
from lxml import etree

start_url = "https://so.gushiwen.cn/shiwen/"
base_url = "https://so.gushiwen.cn"

headers = {
    "user-agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36",
}

items = []


def parse_url(url):
    """解析url，得到响应内容"""
    # time.sleep(random.random())
    response = requests.get(url=url, headers=headers)
    return response.content.decode("utf-8")


def parse_html(html):
    """使用xpath解析html，返回xpath对象"""
    etree_obj = etree.HTML(html)
    return etree_obj


def get_first_type():
    """获取所有的一级类型"""
    first_type_list = []

    html = parse_url(start_url)
    etree_obj = parse_html(html)

    first_type_name_list = etree_obj.xpath('(//a[contains(@href,"/gushi/")]|//a[contains(@href,"/wenyan/")])/text()')
    first_type_url_list = etree_obj.xpath('(//a[contains(@href,"/gushi/")]|//a[contains(@href,"/wenyan/")])/@href')
    data_zip = zip(first_type_name_list, first_type_url_list)

    for data in data_zip:
        first_type = {}
        first_type["name"] = data[0]
        first_type["url"] = data[1]
        first_type_list.append(first_type)

    return first_type_list


def get_data(first_type):
    """查询数据"""

    #一级类型url
    url = base_url + first_type["url"]
    first_type_name = first_type["name"]

    #向一级类型url发送请求获取二级类型数据
    html = parse_url(url)
    etree_obj = parse_html(html)
    div_list = etree_obj.xpath('//div[@class="typecont"]')
    #二级类型类型数据div
    for div in div_list:
        #二级类型名称
        second_type_name = div.xpath(".//strong/text()")
        if second_type_name:  # 有的没有二级类型
            second_type_name = second_type_name[0]
        else:
            second_type_name = ""
        #二级类型下诗词的名称和url
        poetry_name_list = div.xpath(".//span/a/text()")
        poetry_url_list = div.xpath(".//span/a/@href")
        data_zip = zip(poetry_name_list,poetry_url_list)
        for data in data_zip:
            #item是一个诗词数据
            item = {}
            item["first_type_name"] = first_type_name
            item["second_type_name"] = second_type_name
            item["poetry_name"] = data[0]
            #诗词url
            poetry_url = base_url+data[1]
            html = parse_url(poetry_url)
            etree_obj = parse_html(html)
            #诗词作者
            poetry_author = etree_obj.xpath('//p[@class="source"]')[0].xpath(".//text()")
            item["poetry_author"] = "".join(poetry_author).strip()
            #诗词内容
            poetry_content = etree_obj.xpath('//*[@id="contson45c396367f59"]/text()')
            item["poetry_content"] = "".join(poetry_content).strip()
            #诗词译文和注释
            if etree_obj.xpath('//div[@class="contyishang"]'):#有的没有注释
                poetry_explain = etree_obj.xpath('//div[@class="contyishang"]')[0].xpath(".//text()")
                item["poetry_explain"] = "".join(poetry_explain).strip()
            else:
                item["poetry_explain"] = ""
            print(item)
            #保存
            save(item)


def save(item):
    """将数据保存到csv中"""
    with open("./古诗词.csv", "a", encoding="utf-8") as file:
        writer = csv.writer(file)
        writer.writerow(item.values())

def start():
    first_type_list = get_first_type()
    for first_type in first_type_list:
        get_data(first_type)


if __name__ == '__main__':
    start()

三、运行结果

保存数据：