python爬取小说

该代码适用于下面这个小说网站（给这个网站打广告）

小说网

# 网页解析  数据抓取
import os

import requests
from bs4 import BeautifulSoup


def parseHtml(url, fictionName):
    # 设置爬虫的header
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 SE 2.X MetaSr 1.0"}

    # 得到网页内容
    response = requests.get(url, headers=headers)
    # 设置编码
    response.encoding = 'gbk'
    # 得到html
    html = response.text
    # 准备解析
    info = BeautifulSoup(html, "html.parser")
    # 抓取标题
    title = info.find_all('h1', class_="articleH1")
    # 抓取内容
    paragraphs = info.find_all('p')

    title = title[0].text  # 得到标题
    with open(fictionName + '.txt', mode='a', encoding='utf8') as file:
        file.write('\n' + title + '\n')  # 写入标题
        for i in range(len(paragraphs) - 2):
            file.write(paragraphs[i].text + '\n')  # 写入正文


if __name__ == '__main__':
    # 获取第一章URL
    firstChapterURL = input('输入小说第一章的URL：')
    firstChapterURL = firstChapterURL.strip()
    # 获取最后一章URL
    lastChapterURL = input('输入小说最后一章的URL：')
    lastChapterURL = lastChapterURL.strip()
    # 获取文本名称
    fictionName = input('你想给这个文本起什么名字？：')
    # 获取两个URL对应的数字
    firstChapterNumber = int(firstChapterURL.split('/')[-1].split('.')[0])
    lastChapterNumber = int(lastChapterURL.split('/')[-1].split('.')[0])

    # 递减
    if firstChapterNumber >= firstChapterNumber:
        downloadChapterNumber = 1  # 下载第n章
        for i in range(firstChapterNumber, lastChapterNumber - 1, -1):
            url = firstChapterURL.split('.html')[0][:-len(str(firstChapterNumber))] + str(i) + '.html'
            # 爬取网页
            parseHtml(url, fictionName)
            # 显示日志
            print("\033[37;41m 下载第" + str(downloadChapterNumber) + "章完成 \033[0m")
            downloadChapterNumber += 1
    # 递加
    if int(firstChapterNumber) < int(firstChapterNumber):
        downloadChapterNumber = 1
        for i in range(firstChapterNumber, lastChapterNumber + 1, 1):
            url = firstChapterURL.split('.html')[0][:-len(str(firstChapterNumber))] + str(i) + '.html'
            # 爬取网页
            parseHtml(url, fictionName)
            # 显示日志
            print("\033[37;41m 下载第" + str(downloadChapterNumber) + "章完成 \033[0m")
            downloadChapterNumber += 1
    # 告诉文件的位置
    print('文本保存在:\'' + os.getcwd() + '\'    它的名字:\'' + fictionName + '.txt\'')