一、案例介绍
本案例主要使用Python爬取某小说网站的小说,网站已经过(base64)处理:aHR0cHM6Ly93d3cuMjNxYi5jb20v
使用面向对象的方式实现多线程下载,并显示下载进度,从而实现全本下载。
二、实现过程
1.引入库
import requests
from urllib.parse import quote
from lxml import etree
import os
from multiprocessing.dummy import Pool
from tqdm import tqdm
2.流程分析
案例最终需要实现的效果:控制台输入小说名称,显示搜索到的小说列表,可以选择小说序号进行下载,下载的时候显示进度条。
3.代码实现
class novel:
# 将爬虫所需的请求头放在构造方法中 没必要放在这里可以自行获取
def __init__(self):
self.cookies = {}
self.search_headers = {}
self.headers = {}
# 获取小说搜索页面的数据
def search(self):
text = quote(input("输入搜索内容:").encode('gbk'))
data = f'searchkey={text}&searchtype=all'
self.search_res = requests.post('https://www.xxxx.com/saerch.php', cookies=self.cookies,
headers=self.search_headers,
data=data)
self.book_datas()
# 获取单本小说界面的数据
def book_datas(self):
tree = etree.HTML(self.search_res.text)
all_ = tree.xpath('//*[@id="nr"]')
self.datas = []
for i in all_:
data = {
'book_name': i.xpath('.//dt/a/img/@alt')[0],
'book_link': 'https://www.xxxx.com' + i.xpath('.//dt/a/@href')[0]
}
self.datas.append(data)
self.get_chapter()
# 获取小说章节的数据
def get_chapter(self):
for index, item in enumerate(self.datas):
print(f"{index}.", item["book_name"])
choice = int(input("请输入想要下载的小说序号:"))
book_name = self.datas[choice]['book_name']
book_link = self.datas[choice]['book_link']
response = requests.get(book_link, headers=self.headers)
tree = etree.HTML(response.text)
list = tree.xpath('//*[@id="chapterList"]/li')
links = ['https://www.xxxx.com' + i.xpath('.//a/@href')[0] for i in list]
# 添加进度条
total = len(links)
# 参数:长度,名称
self.pdar = tqdm(total=total, desc='下载进度')
if not os.path.exists(f'./{book_name}'):
os.mkdir(f'./{book_name}')
# 定义线程池:提升下载速度
threading_pool = Pool(64)
for index, item in enumerate(links):
if not f'{index}.txt' in os.listdir(f'./{book_name}'):
threading_pool.apply_async(self.download, args=(book_name, index, item,), callback=self.up)
threading_pool.close()
threading_pool.join()
self.merge(book_name, total)
# 下载单章节小说
def download(self, name, index, link):
try:
# print(f'下载第{index}个')
res = requests.get(link, headers=self.headers)
res.encoding = res.apparent_encoding
tree = etree.HTML(res.text)
text = ""
ps = tree.xpath('//*[@id="TextContent"]/p')
for p in ps:
text += p.xpath('.//text()')[0].replace('铅笔小说 23qb.com', '')
with open(f'./{name}/{index}.txt', 'a', encoding='utf-8') as f:
f.write(text)
except FileNotFoundError:
pass
# 更新进度条
def up(self, result):
self.pdar.update(1)
# 将下载的小说合并成一个txt文件
def merge(self, name, total):
for i in range(total):
with open(f"./{name}/{i}.txt", 'r', encoding='utf-8') as fp:
with open(f"./{name}/{name}.txt", 'a', encoding='utf-8') as f:
for line in fp:
f.write(line)
os.remove(f'./{name}/{i}.txt')
if __name__ == '__main__':
novel().search()
4.运行展示
推荐资料:
request库的用法
tqdm库相关
三、总结
例如:以上就是今天要讲的内容,当然代码肯定还会存在些许不足,还需逐步完善。