python基础-爬虫下载小说
import requests
from bs4 import BeautifulSoup
def getnevel(content_url,i):
i=i+1
header = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'}
res = requests.get(content_url,headers = header,timeout = 10)
res.encoding = 'gbk'
soup = BeautifulSoup(res.text,'html.parser')
title = soup.select('b')[0].text.lstrip('章 节目录 ')
content = soup.select('.ART')[0].text.lstrip('style5();').rstrip('style6();')
both = title + content
next_url = 'https://yidukk.com/'+soup.select('.MC .btsc')[2]['href']
print(both,file = f)
print(i)
if(next_url.split('/')[2] != 'yidukk.com' or i>80):
return False
return getnevel(next_url,i)
f = open("《沙雕的成神之路》.txt", 'w+',encoding='utf-8')
i=0
getnevel('https://yidukk.com/read_1045847_32261.html',i)
f.close()
print('ok!')
import requests
from bs4 import BeautifulSoup
#以上作为基本引用
#全局变量
start_url = "https://yidukk.com/read_1045847_32261.html" #小说第一章对应的URL
file_name = "《沙雕的成神之路》.txt" #设置保存的文件名字
max = 80 #最大章节数
header = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'}
count = 0 # 计数器 计数章节数
# function: 获取每章节的小说文字并写入文件中
def getContent(content_url):
global count
count = count +1 #计数器增加
res = requests.get(content_url,headers = header,timeout = 10)
res.encoding = 'gbk'
soup = BeautifulSoup(res.text,'html.parser')
title = soup.select('b')[0].text.lstrip('章 节目录 ') #获取章节题目
content = soup.select('.ART')[0].text.lstrip('style5();').rstrip('style6();') #获取章节内容
both = title + content
print(both,file = f) #写入文件
print("已下载 第"+str(count)+"节") #输出到屏幕提示 状态
next_url = 'https://yidukk.com/'+soup.select('.MC .btsc')[2]['href'] #获取下个章节URL
print(next_url) #测试需要,输出每个章节的URL
if(next_url.split('/')[2] != 'yidukk.com' or count==max):#容错测试,异常抛出,提高稳定性
return False
return getContent(next_url)
#MAIN
if __name__ == '__main__':
f = open(file_name, 'a+',encoding='utf-8')
getContent(start_url)
f.close()
print('小说下载完成!')