python协程处理海量文件_Python使用asyncio和run-In-Executor线程池处理多个文件的同时下载,python,协程,加,runinexecutor...

设置默认线程池

from concurrent.futures import ThreadPoolExecutor

exe = ThreadPoolExecutor(2)

asyncio.get_event_loop().set_default_executor(exe)

同时下载多文件，碰到阻塞的函数则用run_in_executor来新开线程跳过阻塞

# coding=utf-8

import asyncio

import os

import requests

async def downOne(loop, url):

name = os.path.basename(url)

print("downOne:start {}".format(name))

# loop = asyncio.get_event_loop()

# requests.get(url)

result = await loop.run_in_executor(None, requests.get, url)

print("downOne:middle {}".format(name))

if result.status_code == 200:

with open(name, "wb") as fp:

# fp.write(result.content)

r = await loop.run_in_executor(None, fp.write, result.content)

print("downOne: r = {}".format(r))

print("downOne:endxx {}".format(name))

return "ok {}".format(name)

async def deal(loop):

ts = []

for i in range(1, 20):

name = str(i) + ".mp3"

path = "http://mp3-d.ting89.com:9090/有声书/闪婚剩女" + "/" + name

ts.append(asyncio.ensure_future(downOne(loop, path)))

print("deal: middle")

result = await asyncio.gather(*ts)

print("deal:end")

return result

if __name__ == "__main__":

loop = asyncio.get_event_loop()

try:

result = loop.run_until_complete(deal(loop))

print("main:result = {}".format(result))

finally:

loop.close()

print("main end")

结果：

deal: middle

downOne:start 1.mp3

downOne:start 2.mp3

downOne:start 3.mp3

downOne:start 4.mp3

downOne:start 5.mp3

downOne:start 6.mp3

downOne:start 7.mp3

downOne:start 8.mp3

downOne:start 9.mp3

downOne:start 10.mp3

downOne:start 11.mp3

downOne:start 12.mp3

downOne:start 13.mp3

downOne:start 14.mp3

downOne:start 15.mp3

downOne:start 16.mp3

downOne:start 17.mp3

downOne:start 18.mp3

downOne:start 19.mp3

downOne:middle 2.mp3

downOne: r = 4953446

downOne:endxx 2.mp3

downOne:middle 12.mp3

downOne: r = 5028248

downOne:endxx 12.mp3

downOne:middle 3.mp3

downOne: r = 4826282

downOne:endxx 3.mp3

downOne:middle 13.mp3

downOne: r = 5015641

downOne:endxx 13.mp3

downOne:middle 16.mp3

downOne: r = 5011252

downOne:endxx 16.mp3

downOne:middle 17.mp3

downOne: r = 4990877

downOne:endxx 17.mp3

downOne:middle 14.mp3

downOne: r = 4998505

downOne:endxx 14.mp3

downOne:middle 1.mp3

downOne: r = 5231285

downOne:endxx 1.mp3

downOne:middle 11.mp3

downOne: r = 5018113

downOne:endxx 11.mp3

downOne:middle 8.mp3

downOne: r = 4995242

downOne:endxx 8.mp3

downOne:middle 6.mp3

downOne: r = 4956686

downOne:endxx 6.mp3

downOne:middle 4.mp3

downOne: r = 4911755

downOne:endxx 4.mp3

downOne:middle 7.mp3

downOne: r = 4941848

downOne:endxx 7.mp3

downOne:middle 9.mp3

downOne: r = 4909038

downOne:endxx 9.mp3

downOne:middle 5.mp3

downOne: r = 4837358

downOne:endxx 5.mp3

downOne:middle 19.mp3

downOne: r = 5049809

downOne:endxx 19.mp3

downOne:middle 15.mp3

downOne: r = 5006237

downOne:endxx 15.mp3

downOne:middle 10.mp3

downOne: r = 5032441

downOne:endxx 10.mp3

downOne:middle 18.mp3

downOne: r = 5057959

downOne:endxx 18.mp3

deal:end

main:result = ['ok 1.mp3', 'ok 2.mp3', 'ok 3.mp3', 'ok 4.mp3', 'ok 5.mp3', 'ok 6.mp3', 'ok 7.mp3', 'ok 8.mp3', 'ok 9.mp3', 'ok 10.mp3', 'ok 11.mp3', 'ok 12.mp3', 'ok 13.mp3', 'ok 14.mp3', 'ok 15.mp3', 'ok 16.mp3', 'ok 17.mp3', 'ok 18.mp3', 'ok 19.mp3']

main end

PS E:\study\py\async>