"""
author:碧水微澜
url:'https://www.biquwx.la/wanjiexiaoshuo/'#爬取网址
环境:python=3.7
需要的第三方库:parsel、tqdm、requests
target:爬取笔趣阁网站中已完结的小说,并且保存在小说目录下.
"""
import requests
import time
import parsel
from parsel import Selector
from tqdm import tqdm
import os
path='\\'
if not os.path.exists('./小说/'):
os.mkdir('./小说/')#创建小说目录
url=[f'https://www.biquwx.la/wanjiexiaoshuo/{page}' for page in range(1,479+1)]
headers={'Use-Agent':'Mozilla/5.0 (Windows NT 10; Win64; x64; rv:75.0) Gecko/20100101 Firefox/75.0'}
#每一页下小说的地址
def get_response(url):
for url in url:
headers={'Use-Agent':'Mozilla/5.0 (Windows NT 10; Win64; x64; rv:75.0) Gecko/20100101 Firefox/75.0'}
#url=[f'https://www.biquwx.la/wanjiexiaoshuo/{page}' for page in range(1,2+1)]
res=requests.get(url,headers=headers)
res.encoding='gbk'
#print(res.text)
res=Selector(res.text)
return res
break
#get_response(url)
#每本小说章节的url直链
def get_url():
res=get_response(url)
book_name=res.xpath('//*[@id="main"]/div[1]/ul/li/span[2]/a/text()').get()
book_url=res.css('.novelslistss a::attr(href)').extract()
return book_url
#get_url()
#保存函数
def save(name,title,text):
with open('./小说/'+name+'.txt','a',encoding='utf-8')as f:
f.write(title)
f.write('\n\n')
f.write(text)
f.write('\n')
f.close()
#下载小说函数
"""
for i in enumerate(get_url()):
print(i)#每本小说对应索引查询
"""
def get_book():
for url in get_url()[0:10]:#根据索引查看,具体下载哪一本,这里我设置下在第1到11本
res=requests.get(url,headers=headers)
res.encoding='utf-8'
#print(res.text)
sel=Selector(res.text)
#time.sleep(0.5)
urls=sel.css('#list a::attr(href)').extract()#部分章节直链
for urls in tqdm(urls):
all_url=url+urls
res=requests.get(url=all_url,headers=headers)
res.encoding='utf-8'
res=Selector(res.text)
name=res.xpath('//*[@id="wrapper"]/div[5]/div/div[1]/a[3]/text()').get()
title=res.xpath('//*[@id="wrapper"]/div[5]/div/div[2]/h1/text()').get()
content=res.css('#content ::text').getall()
text = ""
for i in content:
text = text + i.strip() + '\n'
save(name,title,text)
if __name__=='__main__':
get_book()
爬取笔趣阁所有完本小说
最新推荐文章于 2024-07-16 18:59:25 发布