【无标题】

最新推荐文章于 2024-09-02 23:14:47 发布

长方体混泥土瞬间移动工程师

最新推荐文章于 2024-09-02 23:14:47 发布

阅读量135

点赞数

分类专栏： xpath 文章标签： python 爬虫开发语言

本文链接：https://blog.csdn.net/allenliuh/article/details/125883064

版权

xpath 专栏收录该内容

2 篇文章 0 订阅

订阅专栏

"""
#!/usr/bin/python3
# -*- coding: utf-8 -*-
# @Time    : 2022/7/17 4:14
# @Author  : allen
"""
from lxml import etree
import requests
from bs4 import BeautifulSoup
import re


if __name__ == "__main__":
    url = "https://www.biqugebang.com/book/4475/"

    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36"}
    response = requests.get(url=url, headers=headers)
    response.encoding = response.apparent_encoding
    page_text = response.text
    tree = etree.HTML(page_text)
    div_list = tree.xpath("//div[@class='listmain']/dl/dd")
    div_list1 = div_list[18:19]
    for div in div_list1:
        text_url = "https://www.biqugebang.com/book/4475/" + div.xpath("./a/@href")[0]
        text_name = div.xpath("./a/text()")[0]
        text_name = text_name.encode('utf-8').decode('utf-8')
        html_data = requests.get(url=text_url, headers=headers).text
        soup1 = BeautifulSoup(html_data, 'html.parser', )
        div_tag = soup1.find("div", id="content")
        content = div_tag.text
        data_list = re.split(r'。', content)
        file = open("a.txt", "a+")
        file.write(text_name+"\n")
        for data in data_list:
            file.write(data + "\n")
        print(text_name, "获取成功！！！")