extract_novel

qq_187352634

已于 2023-11-21 15:17:57 修改

阅读量24

点赞数

分类专栏： spider 文章标签： python beautifulsoup

于 2023-11-12 19:44:00 首次发布

本文链接：https://blog.csdn.net/qq_37755459/article/details/134364609

版权

spider 专栏收录该内容

13 篇文章 0 订阅

订阅专栏

extract_novel

import requests
from bs4 import BeautifulSoup

def chapters():
    base_url = "http://www.timeread.com/book/45274/list"
    resp = requests.get(base_url)
    resp.encoding = "utf-8"
    bf = BeautifulSoup(resp.text, "html.parser")
    data = []
    for li in bf.find("ul", id="htmlChapterList").find_all("li"):
        link_a = li.find("a")
        if not link_a:
            continue
        # print(link_a)
        data.append((link_a["href"], link_a.get_text()))
    return data

def content_novel(url):
    res = requests.get(url)
    res.encoding = "UTF-8"
    bf = BeautifulSoup(res.text, "html.parser")
    if not bf.find("div", id="htmlContent"):
        return None
    return bf.find("div", id="htmlContent").get_text()

novels = chapters()
numbers = len(novels)
ids = 0
for novel_title in chapters():
    url, title = novel_title
    ids += 1
    print("download: "+str(ids)+","+"total:"+ str(numbers))
    with open("%s.txt"%title, 'w', encoding="UTF-8") as fwrite:
        if content_novel(url) ==None:
            continue
        fwrite.write(content_novel(url))
        # print(content_novel(url))
        # break