Python-爬取小说内容并下载

# 文章首页链接
url = "https://www.17k.com/chapter/108821/3148523.html"
def book_spider():
    # 爬取并下载小说内容
    import requests
    from bs4 import BeautifulSoup
    import time
    headers = {"user-agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36"}
    while True:
        global url
        resp = requests.get(url, headers=headers).content
        time.sleep(1)
        html = BeautifulSoup(str(resp, "utf-8"), "lxml")
        # 获取下一章链接
        a_tag = html.find_all("a", class_="nextChapter")
        if len(a_tag) != 2:
            break
        next_tag = a_tag[0]["href"]
        url = "https://www.17k.com" + next_tag
        for tag in html.find_all("div", class_="readAreaBox content"):
            title = tag.find_all("h1")
            # txt文件末尾追加
            with open('小说.txt', 'a+', encoding="utf-8") as f:
                f.write(title[0].string + "\n")  # 标题写入文件
            content = tag.find_all("p")
            for i in content:
                if len(i) == 0 or len(i) == 3:
                    continue
                else:
                    if len(i.string) > 90:
                        with open('小说.txt', "a+", encoding="utf-8") as f:
                            # 分行写入数据，每行最多90个字符串
                            f.writelines("\t%s\n%s" % (i.string[:90], i.string[90:]))
                    else:
                        with open('小说.txt', "a+", encoding="utf-8") as f:
                            f.writelines("\t%s\n" % (i.string))

book_spider()