说明
Python 3.12.1
使用协程的方式来实现下载,速度真的很惊奇!!!
Python 真是方便
import asyncio , aiohttp , aiofiles
import requests
import json , os
# 指定 使用IP4方式, python 默认使用IP6,
# import socket
# import urllib3
# urllib3.util.connection.allowed_gai_family = lambda : socket.AF_INET
# 缺少 第八十一回
# 多个 附录 陈光蕊赴任逢灾 江流僧复仇报本
# start_url = 'https://dushu.baidu.com/api/pc/getCatalog?data={"book_id":"4306063500"}' # 目录页 ok
# url = 'https://dushu.baidu.com/api/pc/getChapterContent?data={"book_id":"4306063500","cid":"4306063500|1569782244","need_bookinfo":1}' # ok
# url = 'https://dushu.baidu.com/api/pc/getChapterContent?data={"book_id":"4306063500","cid":"4306063500|1569782244"}' # ok
async def get_catalog (url):
tasks = []
with requests.get(start_url) as resp:
dic = resp.json()
# items_list = dic["data"]["novel"]["items"]
for dic_item in dic["data"]["novel"]["items"]:
cid = dic_item["cid"]
title = dic_item["title"]
# all_chapter_url_list.append( f'https://dushu.baidu.com/api/pc/getChapterContent?data={{"book_id":"{book_id}","cid":"4306063500|{cid}"}}')
tasks.append( asyncio.create_task(adownload_chapter(book_id,cid,title)))
if max_chapters and len(tasks) == max_chapters:
break
await asyncio.wait(tasks)
async def adownload_chapter(book_id, cid, title):
data = {
"book_id":f"{book_id}",
"cid":f"{book_id}|{cid}"
}
url = f"https://dushu.baidu.com/api/pc/getChapterContent?data={json.dumps(data)}"
# print(url)
async with aiohttp.ClientSession() as session:
async with session.get(url) as resp:
dic = await resp.json()
async with aiofiles.open(f'西游记/{title}.txt',mode='w',encoding='utf-8') as f:
await f.write(dic["data"]["novel"]["content"])
print(f'{title} 下载成功!')
# async def main():
# for url in all_chapter_url_list:
# tasks.append( asyncio.create_task( adownload_chapter(url) ) )
# await asyncio.wait( tasks )
if __name__ == '__main__':
book_name = "西游记"
book_id = 4306063500
start_url = f'https://dushu.baidu.com/api/pc/getCatalog?data={{"book_id":"{book_id}"}}'
# all_chapter_url_list = []
max_chapters = 10
if not os.path.exists(book_name):
os.mkdir(book_name)
asyncio.run( get_catalog(start_url) )