并发数据处理

并发处理数据

经常需要对数据做预处理，当数据量比较大的时候通常处理会比较慢，多线程应该是首先想到能提升速度的方法，这里使用future实现快10倍左右的性能提升。

from concurrent import futures
import glob 
import os
import multiprocessing
import cv2 
import time

def cost_time(fn):
    def wrapper(*args,**kwargs):
        start_time = time.time()
        fn(*args,**kwargs)
        end_time = time.time()
        return "Function:{} cost:{:.4f}s".format(fn.__name__,end_time-start_time)
    return wrapper

def resize_file(files,saved_path='/tmp/resize'):
    for file in files:
        cv2.imwrite(os.path.join(saved_path,os.path.basename(file)),cv2.resize(cv2.imread(file),(299,299)))

@cost_time
def resize_parallal(images_path,saved_path):
    files = list(filter(lambda x:os.path.isfile(x),glob.glob(images_path+"/*/*.jpg",recursive=True)))
    max_work = multiprocessing.cpu_count()
    batch_num = len(files)//max_work
    future_info = {}
    with futures.ThreadPoolExecutor(max_workers=max_work) as executor:
        for i in range(max_work):
            data = files[i*batch_num:(i+1)*batch_num]
            future_info.update({executor.submit(resize_file,data):i})
        for future in futures.as_completed(future_info):
            try:
                data = future.result()
            except Exception as excp:
                print("Exception:{}".format(excp))




def main():
    images_path = "/home/liushuai/flower_photos"
    saved_path = '/tmp/resize'
    print(resize_parallal(images_path,saved_path))
if __name__ == "__main__":
    main()

Function:resize_file cost:16.7528s（单线程）
Function:resize_parallal cost:1.5168s（多线程：24线程）