1.程序说明
说明:御宅书屋小说爬取
网址:https://www.soexs.com/book/188694.html(实例–我在异界建个城)
结果:成功
时间:2024/5/16
‘’’
2.导入第三方库
import requests
from bs4 import BeautifulSoup
import queue
3.小说主页面
#小说主页面--找到小说详细页面链接
def HomeYe(title_url,wz):
# 伪装
res_title = requests.get(title_url, headers=wz)
print('主页面响应:',res_title.status_code)
res_title = res_title.content
# print(res_title)
soup_nr = BeautifulSoup(res_title, 'lxml')
#print(soup_nr)
gengduo=soup_nr.find('div',class_='gengduo')
#print(gengduo)
a=gengduo.find('a')
href_gengduo=a.get('href')
#查看更多章节
href_gengduo=href_gengduo[0:-1]
href_gengduo=http+href_gengduo+'/' #改: 加 /
return href_gengduo
print(href_gengduo)
4.爬取目录分页链接
#爬取目录分页链接
def List(href_gengduo):
res=requests.get(href_gengduo,wz)
print('分页链接响应:',res.status_code)
res = res.content
soup = BeautifulSoup(res, 'lxml')
#print(soup)
fenye=soup.find('div',class_='fy')
#print(fenye)
fenye=fenye.find_all('a')
#print(fenye)
a=len(fenye)/2
a=int(a)-1
fenye=fenye[1:a]
#print(fenye)
#将链接加入队列
q = queue.Queue()
q.put(href_gengduo)
if fenye==[]:
print('没有更多分页了')
else:
for list in fenye:
li = list.get('href')
li=http+li
#print(li)
q.put(li)
print()
return q
5.爬取小说详细章节链接
#爬取小说详细章节链接
def ZhangJie(url,wz):
res_nr = requests.get(url, headers=wz)
print('章节链接响应:',res_nr.status_code) # 检查响应码
res_nr = res_nr.content
# print(res)
soup = BeautifulSoup(res_nr, 'lxml')
#print(soup)
print('列表')
soup=soup.find('ul',class_='lb fk')
#print(soup)
soup=soup.find_all('li')
soup=soup[0:-1]
#print(soup)
p=queue.Queue()
for lis in soup:
hre=lis.find('a',class_='xbk')
#print(hre)
list=hre.get('href')
list=http+list
#print(list)
p.put(list) #队列内只传入了一个值
return p
6.爬取小说内容(每一章)
#小说内容下载
def Dload(p,wz):
res_nr = requests.get(p, headers=wz)
print('章节内容获取响应:',res_nr.status_code) # 检查响应码
res_nr = res_nr.content
# print(res)
soup = BeautifulSoup(res_nr, 'lxml')
# print(soup)
# 本章节内容
# 标题
title = soup.find('div', class_="zhong")
title = title.text
# print("本章标题为:",title)
# 内容
nr = soup.find('article')
nr = nr.text
nr = nr.strip() # 去除字符串前后空格
nr = nr.replace(' ', '')
# print(nr)
str = title + '\n' + nr+'\n'
#文件保存
name='肉要大碗吃'
file = open(f'御书屋/{name}.txt', 'a', encoding='utf-8') # 'a' 为可追加写入
file.write(str) # 路径设置错误 Python写入文件报错‘gbk’ codec can’t encode character的解决办法
file.close()
print(title + "下载完成!")
7.整体运行
if __name__=='__main__':
wz = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/119.0'}
http='https://www.soexs.com'
x=1
title_url = 'https://www.soexs.com/book/188694.html'
HomeYe(title_url=title_url,wz=wz)
href_gengduo=HomeYe(title_url,wz)
print(href_gengduo)
#分页列表获取
List(href_gengduo=href_gengduo)
q=List(href_gengduo)
while not q.empty():
url=q.get()
#print(url)
#章节链接获取
y=ZhangJie(url=url,wz=wz)
# 章节列表获取
while not y.empty():
url_list = y.get()
print('列表:', url_list)
# 章节内容获取
Dload(p=url_list, wz=wz)
print('第{}页'.format(x))
x=x+1