爬取起点小说免费专区-CSDN博客

本文链接：https://blog.csdn.net/x1343676/article/details/141287006

from turtle import title
import requests
from lxml import etree
import json

url = "https://www.qidian.com/chapter/1038058470/767511431/"

headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36 Edg/127.0.0.0",
    "cookie": "supportwebp=true; supportWebp=true; _csrfToken=df5a5e7b-feb3-4658-b9c7-58a2a155fdf2; newstatisticUUID=1709133168_1484421828; fu=380175539; Hm_lvt_f00f67093ce2f38f215010b699629083=1723890564; HMACCOUNT=03CDEC9A006B218E; _gid=GA1.2.227453649.1723890564; e2=; e1=%7B%22l6%22%3A%22%22%2C%22l7%22%3A%22%22%2C%22l1%22%3A3%2C%22l3%22%3A%22%22%2C%22pid%22%3A%22qd_p_qidian%22%2C%22eid%22%3A%22qd_A18%22%7D; traffic_search_engine=; se_ref=; traffic_utm_referer=https%3A%2F%2Fcn.bing.com%2F; Hm_lpvt_f00f67093ce2f38f215010b699629083=1723890756; _ga=GA1.1.2081541013.1688432247; _ga_FZMMH98S83=GS1.1.1723890563.6.1.1723890756.0.0.0; _ga_PFYW0QLV3P=GS1.1.1723890563.6.1.1723890756.0.0.0; w_tsfp=ltvuV0MF2utBvS0Q66nhnUquFDsucDo4h0wpEaR0f5thQLErU5mH0o5zv8v1M3za4cxnvd7DsZoyJTLYCJI3dwMUTMjFe9wZjgXDkIAi2YgQBhljR8qOCgRJJ+4k5GNDKHhCNxS00jA8eIUd379yilkMsyN1zap3TO14fstJ019E6KDQmI5uDW3HlFWQRzaLbjcMcuqPr6g18L5a5Tba7QiqeF0iALMR1EbB0SFNDHoj4Ee+d+8OMhukcp+sSqA=",
}
response = requests.get(url, headers=headers)

content = response.text

html = etree.HTML(content, etree.HTMLParser())

json_data = html.xpath("//script[@id='vite-plugin-ssr_pageContext']/text()")[0]
# 转换json数据变成python字典对象
data = json.loads(json_data)

# pageContext-->pageProps-->pageData-->chapterInfo-->content层级结构
chapterName = data["pageContext"]["pageProps"]["pageData"]["chapterInfo"]["chapterName"]
content = data["pageContext"]["pageProps"]["pageData"]["chapterInfo"]["content"]

# 去除<p>标签,变成换行符
content = content.replace("<p>", "\n")
print(content)