一个简单的爬小说的例子

YouYOUyouFairy

于 2024-04-24 09:35:22 发布

阅读量92

点赞数 2

文章标签：爬虫

本文链接：https://blog.csdn.net/fairy_yang_1/article/details/138146585

版权

import requests
import re
import lxml.html



url = 'https://www.hongxiu.com/book/16992320705883804'
headers = {
    "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"
}

r = requests.get(url,headers)
# print(r.text)

parr = re.compile('<li data-rid="(.*?)"><a href="(.*?)" target="_blank">(.*?)</a>')
titles = re.findall(parr, r.text)
with open('xiaoshuo.txt','w',encoding='utf-8') as f:
    for title in titles:
        print("第"+ title[0]+"章"+' '+title[2])
        f.write("第"+ title[0]+"章"+' '+title[2])
        f.write('\n')
        chapt_url='https://www.hongxiu.com'+ title[1]
        # print(chapt_url)
        rr = requests.get(chapt_url,headers=headers).content.decode()
        selector = lxml.html.fromstring(rr)
        info = selector.xpath('//div[@class="ywskythunderfont"]/p/text()')
        for i in info:
            f.write(i)
            f.write('\n')
            print(i)