【自用】Python爬虫学习（五）：协程asyncio、aiohttp下载多幅图片与小说

---

使用协程下载多幅图片（asyncio、aiohttp）

import asyncio
import aiohttp
import time

# 用协程下载多幅图片

urls = ["https://www.umei.cc/d/file/20230906/9503a3c2752d24f896fc12706a54c9ab.jpg",
        "https://www.umei.cc/d/file/20230906/50bec42cb5ca371a3ddc715c380ab91d.jpg",
        "https://www.umei.cc/d/file/20230823/3aa0b5ac78d922916148db687cb17a66.jpg"]


async def aiodownload(url):
    name = url.rsplit("/", 1)[1]  # split加了个r表示反转，从右边切，得到[1]的位置
    async with aiohttp.ClientSession() as session:
        async with await session.get(url) as resp:
            with open("img/" + name, mode="wb") as f:
                # 请求回来了，写入文件
                f.write(await resp.content.read())  # 读取内容是异步的：需要await挂起
    print(name, "已下载！")


async def main():
    tasks = []
    for url in urls:
        tasks.append(asyncio.create_task(aiodownload(url)))
    await asyncio.wait(tasks)


if __name__ == '__main__':
    start = time.time()
    # # 执行函数写法1，python 3.7版本以前版本
    # loop = asyncio.get_event_loop()
    # loop.run_until_complete(main())

    # 执行函数写法2，支持python 3.7版本以后
    asyncio.run(main())
    end = time.time()
    print("耗时：", end - start)

使用协程下载小说

import requests
import asyncio
import aiohttp
import json

# 下载某度小说中的《西游记》

"""
1.同步操作：访问getCatalog 拿到所有章节的cid和名称
2.异步操作：访问getchapterContent 下载所有的文章内容
"""


async def aiodownload(b_id, cid, title):
    data = {"book_id": b_id,
            "cid": f"{b_id}|{cid}",
            "need_bookinfo": 1}
    data = json.dumps(data)  # 转为json字符串
    url_chapter = f'https://dushu.baidu.com/api/pc/getChapterContent?data={data}'
    async with aiohttp.ClientSession() as session:
        async with await session.get(url_chapter) as resp:
            # text())返回字符串形式的响应数据
            # read()返回的二进制形式的响应数据
            # json()返回的就是json对象
            dict = await resp.json()  # 获取响应数据操作之前必须手动进行await挂起，否则拿不到对应的数据
            with open('data_file/novel/' + title + '.csv', mode='w', encoding='utf-8') as f:
                # 请求回来了，将每章节对应的内容写入对应的csv文件，csv文件名为章节标题
                # 注意，这里文件写入也应该用aiofiles对应的写入函数
                # 因为当时这个模块没安装成功，就先使用普通的写入函数，这会导致效率下降
                f.write(dict['data']['novel']['content'])
                print(f"'{title}.csv'\t写入成功！")


async def getCatalog(url):
    resp = requests.get(url)
    # print(resp.json())

    dict = resp.json()
    tasks = []
    b_id = dict['data']['novel']['book_id']
    for item in dict['data']['novel']['items']:
        title = item['title']
        cid = item['cid']
        # print(cid, title)

        # 准备异步任务
        tasks.append(asyncio.create_task(aiodownload(b_id, cid, title)))

    await asyncio.wait(tasks)


if __name__ == '__main__':
    # 获取所有章节的名称，cid
    url_catalog = 'https://dushu.baidu.com/api/pc/getCatalog?data={%22book_id%22:%224306063500%22}'
    # getCatalog(url_catalog)

    asyncio.run(getCatalog(url_catalog))
    print('所有章节下载完成！')