python异步编程案例之协程并发

有几种常用协程并发的方式，asyncio.wait 和 asyncio.gather，以及asyncio.as_completed。

asyncio.gather方法能够收集协程的结果(只告诉你协程结果，gather 支持return_exceptions参数)，它会按输入协程的顺序保存的对应协程的执行结果。

asyncio.wait的返回值有 2 项，第一项表示完成的任务列表 (done)，第二项表示等待 (Future) 完成的任务列表 (pending)，每个任务都是一个 Task 实例（await支持return_when参数）。如果你关注协程执行结果你需要从对应 Task 实例里面用 result 方法自己拿。在默认情况下，asyncio.wait会等待全部任务完成 (return_when=‘ALL_COMPLETED’)。它还支持 FIRST_COMPLETED（第一个协程完成就返回）和 FIRST_EXCEPTION（出现第一个异常就返回）。它不能保证按输入协程的顺序保存的对应协程的执行结果

asyncio.as_completed 每个协程结束运行时一次生成一个结果. 不同于wait,gather之处在于，as_competed, 会在当前任务执行完成后立即返回结果，不会等待所有后台操作完成。与 wait 一样，as_complete 不能保证顺序

asyncio.wait，在默认情况下，asyncio.wait会等待全部任务完成 (return_when=‘ALL_COMPLETED’)，

import asyncio
async def test():
    print('Hello test')
    await asyncio.sleep(1)
    print('Hello again test, ', x)
    return "test1"
async def test2():
    print('Hello test2')
    await asyncio.sleep(2)
    print('Hello again test2')
    return "test2"
done, pending = await asyncio.wait([test(), test2()], return_when=asyncio.tasks.FIRST_COMPLETED)

Hello test
Hello test2
Hello again test2

test() 完成，test2()仍然处于pending状态

done
Out[19]: {<Task finished coro=<async-def-wrapper.<locals>.test() done, defined at <ipython-input-18-3ea1ddd4ae45>:5> exception=NameError("name 'x' is not defined",)>}
pending
Out[20]: {<Task pending coro=<async-def-wrapper.<locals>.test2() running at <ipython-input-18-3ea1ddd4ae45>:12> wait_for=<Future pending cb=[<TaskWakeupMethWrapper object at 0x0000020B26ADF228>()]>>}

下面代码中，我分别将三种不同的挂起方式封装到不同的协程函数里，分别为 hang_coroutine_wait，hang_coroutine_gather， hang_coroutine_as_completed。

import time
import asyncio

now = lambda : time.time()

async def do_some_work(x):
    print("Hello:",x)
    return "work is done for {}".format(x)
    
async def hang_coroutine_wait(tasks):
    dones, pendings = await asyncio.wait(tasks)
    for task in dones:
        print("task result:", task.result())
        
async def hang_coroutine_gather(tasks):
    results = await asyncio.gather(*tasks)
    for result in results:
        print("Task result:",result)
        
async def hang_coroutine_as_completed(tasks):
    for task in asyncio.as_completed(tasks):
        result = await task
        print('Task result:{}'.format(result))
        
async def main():
    tasks = []
    for i in range(5):
        coroutine = do_some_work(i)
        # task = asyncio.ensure_future(coroutine)
        task = loop.create_task(coroutine)
        tasks.append(task)
        
    # await hang_coroutine_as_completed(tasks)
    await hang_coroutine_wait(tasks)
    # await hang_coroutine_gather(tasks)
    
start = now()
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop.run_until_complete(main()) # 将协程注册到事件循环，并启动事件循环
loop.close()   
end = now()
print("total run time:", end-start)

Hello: 0
Hello: 1
Hello: 2
Hello: 3
Hello: 4
task result: work is done for 3
task result: work is done for 0
task result: work is done for 4
task result: work is done for 2
task result: work is done for 1
total run time: 0.0009984970092773438

如果对协程嵌套不熟悉，可以看下面代码。(其实下面也是嵌套的，只不过没有封装挂起。。。)

import time
import asyncio

now = lambda : time.time()

async def do_some_work(x):
    print("Hello:",x)
    return "work is done for {}".format(x)
    
async def main():
    tasks = []
    for i in range(5):
        coroutine = do_some_work(i)
        # task = asyncio.ensure_future(coroutine)
        task = loop.create_task(coroutine)
        tasks.append(task)
        
    dones, pendings = await asyncio.wait(tasks)
    for task in dones:
        print("task result:", task.result())
        
    # results = await asyncio.gather(*tasks)
    # for result in results:
    #     print("Task result:",result)
    
    # for task in asyncio.as_completed(tasks):
    #     result = await task
    #     print('Task result:{}'.format(result))
        
start = now()
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop.run_until_complete(main()) # 将协程注册到事件循环，并启动事件循环
loop.close()   
end = now()
print("total run time:", end-start)

Hello: 0
Hello: 1
Hello: 2
Hello: 3
Hello: 4
task result: work is done for 2
task result: work is done for 3
task result: work is done for 4
task result: work is done for 1
task result: work is done for 0
total run time: 0.001999378204345703

也可以直接返回await的内容（可以是asyncio.wait, 也可以是asyncio.gather），不在main协程函数里处理结果。外层的run_until_complete返回main协程的结果。
await asyncio.wait

import asyncio
import time

now = lambda: time.time()

async def do_some_work(x):
    print("Hello:", x)
    return "work is done for {}".format(x)
    
async def main():
    tasks = []
    for i in range(5):
        coroutine = do_some_work(i)
        # task = asyncio.ensure_future(coroutine)
        task = loop.create_task(coroutine)
        tasks.append(task)
    return await asyncio.wait(tasks)
    
start = now()
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
dones, pendings = loop.run_until_complete(main())
for task in dones:
    print("task result:",task.result())
print("total run time:", now()-start)

Hello: 0
Hello: 1
Hello: 2
Hello: 3
Hello: 4
task result: work is done for 2
task result: work is done for 3
task result: work is done for 0
task result: work is done for 4
task result: work is done for 1
total run time: 0.0019948482513427734

或者await asyncio.gather

import asyncio
import time

now = lambda: time.time()

async def do_some_work(x):
    print("Hello:", x)
    return "work is done for {}".format(x)
    
async def main():
    tasks = []
    for i in range(5):
        coroutine = do_some_work(i)
        # task = asyncio.ensure_future(coroutine)
        task = loop.create_task(coroutine)
        tasks.append(task)
    return await asyncio.gather(*tasks)
start = now()
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
results = loop.run_until_complete(main())
for result in results:
    print("task result:",result)
print("total run time:", now()-start)

Hello: 0
Hello: 1
Hello: 2
Hello: 3
Hello: 4
task result: work is done for 0
task result: work is done for 1
task result: work is done for 2
task result: work is done for 3
task result: work is done for 4
total run time: 0.001966238021850586