concurrent.futures和ProcessPoolExecutor这两个类实现的借口分别在不同的线程或进程中执行可调用的对象,这两个类在内部维护者一个工作线程或进程池,以及要执行的队列,这两个借口抽象的层级很高,无需关注实现细节
普通方法实现下载国旗
import os
import time
import sys
import requests
POP20_CC=('CN IN US ID BR PK NG BD RU JP'
'MX PH VN ET EG DE IR TR CD FR').split()
BASE_URL='http://flupy.org/data/flags'
DEST_DIR='downloads/'
def save_flag(img,filename):
path=os.path.join(DEST_DIR,filename)
with open(path,'wb')as fp:
fp.write(img)
def get_flag(cc):
url='{}/{cc}/{cc}.gif'.format(BASE_URL,cc=cc.lower())
resp=requests.get(url)
return resp.content
def show(text):
print(text,end="")
sys.stdout.flush()
def download_many(cc_list):
for cc in sorted(cc_list):
image=get_flag(cc)
show(cc)
save_flag(image,cc.lower()+'.gif')
return len(cc_list)
def main(download_many):
to=time.time()
count=download_many(POP20_CC)
elapsed=time.time()
msg='\n{} flags downloaded in {:.2f}s'
print(msg.format(count,elapsed))
if __name__=='__main__':
main(download_many)
替代多线程方法
#!/usr/bin/env python
# -*- coding:utf-8 -*-
from concurrent import futures
import sys
# sys.path.append(r"F:\regtest\asyncio!")
# from qw import save_flag,get_flag,show ,main
import os
import time
import sys
MAX_WORKERS=20
import requests
POP20_CC=('CN IN US ID BR PK NG BD RU JP'
'MX PH VN ET EG DE IR TR CD FR').split()
BASE_URL='http://flupy.org/data/flags'
DEST_DIR='downloads/'
def save_flag(img,filename):
path=os.path.join(DEST_DIR,filename)
with open(path,'wb')as fp:
fp.write(img)
def get_flag(cc):
url='{}/{cc}/{cc}.gif'.format(BASE_URL,cc=cc.lower())
resp=requests.get(url)
return resp.content
def show(text):
print(text,end="")
sys.stdout.flush()
def download_one(cc):
image=get_flag(cc)
show(cc)
save_flag(image,cc.lower()+'.gif')
return cc
def download_many(cc_list):
workers=min(MAX_WORKERS,len(cc_list))
with futures.ThreadPoolExecutor(workers) as executor:
res=executor.map(download_one,sorted(cc_list))
return len(list(res))
def main(download_many):
to=time.time()
count=download_many(POP20_CC)
elapsed=time.time()
msg='\n{} flags downloaded in {:.2f}s'
print(msg.format(count,elapsed))
if __name__=='__main__':
main(download_many)
有了这个神器就再也不用耗费精力去创建线程池或者进程池,以及队列来处理问题了。~
关于如何创建线程池进程池,在python基础篇~