python 协程加多线程下载asyncio、ThreadPoolExecutor

使用协程同时下载多文件，碰到阻塞函数则无法同时下载多文件，结合多线程处理。

# coding=utf-8

import asyncio
import os
import requests
from concurrent.futures import ThreadPoolExecutor

gPool = ThreadPoolExecutor(5)


async def downOne(loop, url):
    name = os.path.basename(url)
    print("downOne:start {}".format(name))
    # loop = asyncio.get_event_loop()
    # requests.get(url)
    global gPool
    result = await loop.run_in_executor(gPool, requests.get, url)
    print("downOne:middle {}".format(name))
    if result.status_code == 200:
        with open(name, "wb") as fp:
            # fp.write(result.content)

            # r = await loop.run_in_executor(None, fp.write, result.content)
            r = await loop.run_in_executor(gPool, fp.write, result.content)
            print("downOne: r = {}".format(r))
    print("downOne:endxx {}".format(name))
    return "ok {}".format(name)


async def deal(loop):
    ts = []
    for i in range(1, 20):
        name = str(i) + ".mp3"
        path = "http://mp3-d.ting89.com:9090/有声书/闪婚剩女" + "/" + name
        ts.append(asyncio.ensure_future(downOne(loop, path)))
    print("deal: middle")
    result = await asyncio.gather(*ts)
    print("deal:end")
    return result


if __name__ == "__main__":
    loop = asyncio.get_event_loop()

    try:
        result = loop.run_until_complete(deal(loop))
        print("main:result = {}".format(result))
    finally:
        loop.close()

    print("main end")

结果：

deal: middle
downOne:start 1.mp3
downOne:start 2.mp3
downOne:start 3.mp3
downOne:start 4.mp3
downOne:start 5.mp3
downOne:start 6.mp3
downOne:start 7.mp3
downOne:start 8.mp3
downOne:start 9.mp3
downOne:start 10.mp3
downOne:start 11.mp3
downOne:start 12.mp3
downOne:start 13.mp3
downOne:start 14.mp3
downOne:start 15.mp3
downOne:start 16.mp3
downOne:start 17.mp3
downOne:start 18.mp3
downOne:start 19.mp3
downOne:middle 2.mp3
downOne:middle 5.mp3
downOne:middle 1.mp3
downOne:middle 4.mp3
downOne:middle 3.mp3
downOne:middle 7.mp3
downOne:middle 6.mp3
downOne:middle 8.mp3
downOne:middle 9.mp3
downOne:middle 11.mp3
downOne:middle 13.mp3
downOne:middle 10.mp3
downOne:middle 16.mp3
downOne:middle 12.mp3
downOne:middle 15.mp3
downOne: r = 4953446
downOne:endxx 2.mp3
downOne: r = 4837358
downOne:endxx 5.mp3
downOne: r = 5231285
downOne:endxx 1.mp3
downOne: r = 4911755
downOne:endxx 4.mp3
downOne: r = 4826282
downOne:endxx 3.mp3
downOne: r = 4941848
downOne:endxx 7.mp3
downOne: r = 4956686
downOne:endxx 6.mp3
downOne: r = 4995242
downOne:endxx 8.mp3
downOne: r = 4909038
downOne:endxx 9.mp3
downOne: r = 5018113
downOne:endxx 11.mp3
downOne: r = 5015641
downOne:endxx 13.mp3
downOne: r = 5032441
downOne:endxx 10.mp3
downOne: r = 5011252
downOne:endxx 16.mp3
downOne: r = 5028248
downOne:endxx 12.mp3
downOne: r = 5006237
downOne:endxx 15.mp3
downOne:middle 14.mp3
downOne: r = 4998505
downOne:endxx 14.mp3
downOne:middle 17.mp3
downOne: r = 4990877
downOne:endxx 17.mp3
downOne:middle 19.mp3
downOne: r = 5049809
downOne:endxx 19.mp3
downOne:middle 18.mp3
downOne: r = 5057959
downOne:endxx 18.mp3
deal:end
main:result = ['ok 1.mp3', 'ok 2.mp3', 'ok 3.mp3', 'ok 4.mp3', 'ok 5.mp3', 'ok 6.mp3', 'ok 7.mp3', 'ok 8.mp3', 'ok 9.mp3', 'ok 10.mp3', 'ok 11.mp3', 'ok 12.mp3', 'ok 13.mp3', 'ok 14.mp3', 'ok 15.mp3', 'ok 16.mp3', 'ok 17.mp3', 'ok 18.mp3'