9_协程及应用

import asyncio


async def func1():
    print("你好啊, 我叫潘金莲")
    await asyncio.sleep(3)
    print("你好啊, 我叫潘金莲")


async def func2():
    print("你好啊, 我叫王建国")
    await asyncio.sleep(2)
    print("你好啊, 我叫王建国")


async def func3():
    print("你好啊, 我叫李雪琴")
    await asyncio.sleep(4)
    print("你好啊, 我叫李雪琴")


async def main():
    # 第一种写法
    # f1 = func1()
    # await f1  # 一般await挂起操作放在协程对象前面
    # 第二种写法(推荐)
    tasks1 = [
        func1(),  
        func2(),
        func3()
    ]
    tasks2 = [
        asyncio.create_task(func1()),  # py3.8以后加上asyncio.create_task()
        asyncio.create_task(func2()),
        asyncio.create_task(func3())
    ]
    await asyncio.wait(tasks1)


if __name__ == '__main__':
    # 一次性启动多个任务(协程)
    asyncio.run(main())


##################线程+多线程+协程三种方式下载图片应用
# requests.get() 同步的代码 -> 异步操作aiohttp
# pip install aiohttp
import time
import asyncio
import aiohttp

from threading import Thread
import requests

from concurrent.futures import ThreadPoolExecutor

urls = [
    "http://kr.shanghai-jiuxin.com/file/2020/1031/191468637cab2f0206f7d1d9b175ac81.jpg",
    "http://kr.shanghai-jiuxin.com/file/2020/1031/563337d07af599a9ea64e620729f367e.jpg",
    "http://kr.shanghai-jiuxin.com/file/2020/1031/774218be86d832f359637ab120eba52d.jpg"
]


async def aiodownload(url):
    # 发送请求.
    # 得到图片内容
    # 保存到文件
    name = url.rsplit("/", 1)[1]  # 从右边切, 切一次. 得到[1]位置的内容
    async with aiohttp.ClientSession() as session:  # requests
        async with session.get(url) as resp:  # resp = requests.get()
            # 请求回来了. 写入文件
            with open("./pic/"+name, mode="wb") as f:  # 创建文件
                f.write(await resp.content.read())  # 读取内容是异步的. 需要await挂起, resp.text()

    print(name, "搞定")


async def main():
    tasks = []
    for url in urls:
        #python 3.8版本以前这么用
        tasks.append(aiodownload(url))
        #ptyhon3.8以后这么用
        # tasks.append(asyncio.create_task(aiodownload(url)))
        
    await asyncio.wait(tasks)


def duoxiancheng_download(url):
    resp = requests.get(url)
    name = url.rsplit("/", 1)[1]
    with open("./pic/"+name, mode="wb") as f:  # 创建文件
        f.write(resp.content)  # 读取内容是异步的. 需要await挂起, resp.text()

    print(name, "搞定")


if __name__ == '__main__':
    #协程版本
    # t1 = time.time()
    # asyncio.run(main())
    # t2 = time.time()
    # print(t2-t1)  #2.6049141883850098

    #多线程版本
    # t1 = time.time()
    # for i in range(3):
    #     temp = Thread(target=duoxiancheng_download, args=(urls[i],))
    #     temp.start()
    # # d1 = Thread(target=duoxiancheng_download, args=(urls[0],))
    # # d1.start()
    # # d2 = Thread(target=duoxiancheng_download, args=(urls[1],))
    # # d2.start()
    # # d3 = Thread(target=duoxiancheng_download, args=(urls[2],))
    # # d3.start()
    # t2 = time.time()
    # print(t2-t1)

    #利用线程池的多线程版本
    t1 = time.time()
    with ThreadPoolExecutor(5) as t:
        for i in range(3):  
            # 把下载任务提交给线程池
            # t.submit(duoxiancheng_download, urls[i])
            t.submit(duoxiancheng_download, url=urls[i])
    t2 = time.time()
    print(t2-t1)  #3.0072884559631348



############################用协程爬小说
# http://dushu.baidu.com/api/pc/getCatalog?data={"book_id":"4306063500"}  => 所有章节的内容(名称, cid)
# 章节内部的内容
# http://dushu.baidu.com/api/pc/getChapterContent?data={"book_id":"4306063500","cid":"4306063500|11348571","need_bookinfo":1}

import requests
import asyncio
import aiohttp
import aiofiles
import json
import time

"""
1. 同步操作: 访问getCatalog 拿到所有章节的cid和名称
2. 异步操作: 访问getChapterContent 下载所有的文章内容
"""

#2. 异步操作:
async def aiodownload(cid, b_id, title):
    data = {
        "book_id":b_id,
        "cid":f"{b_id}|{cid}",
        "need_bookinfo":1
    }
    #用于拼凑章节内容的请求网址
    data = json.dumps(data)
    url = f"http://dushu.baidu.com/api/pc/getChapterContent?data={data}"

    async with aiohttp.ClientSession() as session:
        async with session.get(url) as resp:
            dic = await resp.json()

            #不使用异步写入  3.5019617080688477
            # with open("./novel1/"+title, mode="w", encoding="utf-8") as f:  # 创建文件
            #     f.write(dic['data']['novel']['content'])

            #使用异步写入  1.4040799140930176
            async with aiofiles.open("./novel1/"+title, mode="w", encoding="utf-8") as f:
                await f.write(dic['data']['novel']['content'])  # 把小说内容写出

#1. 同步操作:
async def getCatalog(url):
    resp = requests.get(url)
    #将获取到的页面目录数据转化为字典形式的数据，便于取数据
    dic = resp.json()
    tasks = []
    for item in dic['data']['novel']['items']:  # item就是对应每一个章节的名称和cid
        title = item['title']
        cid = item['cid']
        # 准备异步任务
        tasks.append(aiodownload(cid, b_id, title))
    #开启协程异步任务
    await asyncio.wait(tasks)


if __name__ == '__main__':
    t1 = time.time()
    b_id = "4306063500"
    url = 'http://dushu.baidu.com/api/pc/getCatalog?data={"book_id":"' + b_id + '"}'
    asyncio.run(getCatalog(url))
    t2 = time.time()
    print(t2-t1)