爬取笔趣阁所有完本小说

"""
author:碧水微澜
url:'https://www.biquwx.la/wanjiexiaoshuo/'#爬取网址
环境：python=3.7
需要的第三方库：parsel、tqdm、requests
target：爬取笔趣阁网站中已完结的小说，并且保存在小说目录下.

"""
import requests
import time
import parsel
from parsel import Selector
from tqdm import tqdm
import os
path='\\'
if not os.path.exists('./小说/'):
    os.mkdir('./小说/')#创建小说目录
url=[f'https://www.biquwx.la/wanjiexiaoshuo/{page}' for page in range(1,479+1)]
headers={'Use-Agent':'Mozilla/5.0 (Windows NT 10; Win64; x64; rv:75.0) Gecko/20100101 Firefox/75.0'}

#每一页下小说的地址
def get_response(url):
    for url in url:
        headers={'Use-Agent':'Mozilla/5.0 (Windows NT 10; Win64; x64; rv:75.0) Gecko/20100101 Firefox/75.0'}
        #url=[f'https://www.biquwx.la/wanjiexiaoshuo/{page}' for page in range(1,2+1)]
        res=requests.get(url,headers=headers)
        res.encoding='gbk'
        #print(res.text)
        res=Selector(res.text)
        return res
        break
#get_response(url)
#每本小说章节的url直链
def get_url():
    res=get_response(url)
    book_name=res.xpath('//*[@id="main"]/div[1]/ul/li/span[2]/a/text()').get()
    book_url=res.css('.novelslistss a::attr(href)').extract()
    return book_url
#get_url()
#保存函数
def save(name,title,text):
    with open('./小说/'+name+'.txt','a',encoding='utf-8')as f:
        f.write(title)
        f.write('\n\n')
        f.write(text)
        f.write('\n')
        f.close()
#下载小说函数

"""
for i in enumerate(get_url()):
    print(i)#每本小说对应索引查询

"""
def get_book(): 
    for url in get_url()[0:10]:#根据索引查看，具体下载哪一本，这里我设置下在第1到11本
   
        res=requests.get(url,headers=headers)
        res.encoding='utf-8'
        #print(res.text)
        sel=Selector(res.text)
        #time.sleep(0.5)
        urls=sel.css('#list a::attr(href)').extract()#部分章节直链
        for urls in tqdm(urls):
            all_url=url+urls
            res=requests.get(url=all_url,headers=headers)
            res.encoding='utf-8'
            res=Selector(res.text)
            name=res.xpath('//*[@id="wrapper"]/div[5]/div/div[1]/a[3]/text()').get()
            title=res.xpath('//*[@id="wrapper"]/div[5]/div/div[2]/h1/text()').get()
            content=res.css('#content ::text').getall()
            text = ""
            for i in content:
                text = text + i.strip() + '\n'
            save(name,title,text)

if __name__=='__main__':
    get_book()