python基础-爬虫下载小说

import requests
from bs4 import BeautifulSoup


def getnevel(content_url,i):
    i=i+1
    header = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'}
    res = requests.get(content_url,headers = header,timeout = 10)
    res.encoding = 'gbk'
    soup = BeautifulSoup(res.text,'html.parser')
    title = soup.select('b')[0].text.lstrip('章 节目录 ')
    content = soup.select('.ART')[0].text.lstrip('style5();').rstrip('style6();')
    both = title + content
    next_url = 'https://yidukk.com/'+soup.select('.MC .btsc')[2]['href']
    print(both,file = f)
    print(i)
    if(next_url.split('/')[2] != 'yidukk.com' or i>80):
        return False
    return getnevel(next_url,i)


f = open("《沙雕的成神之路》.txt", 'w+',encoding='utf-8')
i=0
getnevel('https://yidukk.com/read_1045847_32261.html',i)
f.close()
print('ok!')

import requests
from bs4 import BeautifulSoup
#以上作为基本引用


#全局变量
start_url = "https://yidukk.com/read_1045847_32261.html" #小说第一章对应的URL
file_name = "《沙雕的成神之路》.txt"  #设置保存的文件名字
max = 80  #最大章节数

header = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'}
count = 0 # 计数器 计数章节数


# function： 获取每章节的小说文字并写入文件中
def getContent(content_url):

    global count
    count = count +1 #计数器增加
    
    res = requests.get(content_url,headers = header,timeout = 10)
    res.encoding = 'gbk'
    
    soup = BeautifulSoup(res.text,'html.parser')
    title = soup.select('b')[0].text.lstrip('章 节目录 ') #获取章节题目
    content = soup.select('.ART')[0].text.lstrip('style5();').rstrip('style6();') #获取章节内容
    both = title + content

    print(both,file = f) #写入文件
    print("已下载 第"+str(count)+"节") #输出到屏幕提示 状态
    
    next_url = 'https://yidukk.com/'+soup.select('.MC .btsc')[2]['href']  #获取下个章节URL
    print(next_url) #测试需要，输出每个章节的URL
    if(next_url.split('/')[2] != 'yidukk.com' or count==max):#容错测试，异常抛出，提高稳定性
        return False
    return getContent(next_url)


#MAIN
if __name__ == '__main__':
    f = open(file_name, 'a+',encoding='utf-8')
    getContent(start_url)
    f.close()
    print('小说下载完成!')