[Python] 协程下载百度小说_西游记

说明

Python 3.12.1
使用协程的方式来实现下载,速度真的很惊奇!!!
Python 真是方便

import asyncio , aiohttp , aiofiles
import requests
import json , os

# 指定 使用IP4方式, python 默认使用IP6,
# import socket
# import urllib3
# urllib3.util.connection.allowed_gai_family = lambda : socket.AF_INET

# 缺少 第八十一回 
# 多个 附录 陈光蕊赴任逢灾 江流僧复仇报本

# start_url = 'https://dushu.baidu.com/api/pc/getCatalog?data={"book_id":"4306063500"}'  # 目录页  ok
# url =       'https://dushu.baidu.com/api/pc/getChapterContent?data={"book_id":"4306063500","cid":"4306063500|1569782244","need_bookinfo":1}'  # ok
# url =       'https://dushu.baidu.com/api/pc/getChapterContent?data={"book_id":"4306063500","cid":"4306063500|1569782244"}'    # ok


async def get_catalog (url):
    tasks = []
    with requests.get(start_url) as resp:
        dic = resp.json()
        # items_list = dic["data"]["novel"]["items"]
        for dic_item in dic["data"]["novel"]["items"]:
            cid = dic_item["cid"]
            title = dic_item["title"]
            # all_chapter_url_list.append( f'https://dushu.baidu.com/api/pc/getChapterContent?data={{"book_id":"{book_id}","cid":"4306063500|{cid}"}}')
            tasks.append( asyncio.create_task(adownload_chapter(book_id,cid,title)))
            if max_chapters and len(tasks) == max_chapters:
                break

    await asyncio.wait(tasks)
        


async def adownload_chapter(book_id, cid, title):
    data = {
        "book_id":f"{book_id}",
        "cid":f"{book_id}|{cid}"
    }
    url = f"https://dushu.baidu.com/api/pc/getChapterContent?data={json.dumps(data)}"
    # print(url)
    async with aiohttp.ClientSession() as session:
        async with session.get(url)  as resp:
            dic = await resp.json()
            async with aiofiles.open(f'西游记/{title}.txt',mode='w',encoding='utf-8') as f:
                await f.write(dic["data"]["novel"]["content"])
                print(f'{title} 下载成功!')



# async  def main():
    
#     for url in all_chapter_url_list:
#         tasks.append( asyncio.create_task(  adownload_chapter(url) ) )

#     await asyncio.wait( tasks )


if __name__ == '__main__':
    book_name = "西游记"
    book_id = 4306063500
    start_url = f'https://dushu.baidu.com/api/pc/getCatalog?data={{"book_id":"{book_id}"}}'
    # all_chapter_url_list = []
    
    max_chapters = 10

    if not os.path.exists(book_name):
        os.mkdir(book_name)

    asyncio.run( get_catalog(start_url) )