python并行

concurrent：built-in module，效率不错　　

def calculate(arg, num_process=8):
    if type(arg) is list or isinstance(arg, types.GeneratorType):
        # Approach 1: Use Python ProcessPoolExecutor
        with concurrent.futures.ProcessPoolExecutor(num_process) as executor:
            result_list = list(executor.map(calculate_single_scenario, arg))
        return result_list
    else:
        return calculate_single_scenario(arg)

scoop: map, reduce, 调用多台机器

joblib：并行，且cache以前算过的。

import joblib
from joblib import Parallel, delayed
from joblib import Memory
def calculate_with_cache(arg, num_process=8):
    func_cached = memory.cache(calculate_single_scenario)
if type(arg) is list or isinstance(arg, types.GeneratorType):
        return Parallel(n_jobs=num_process)(delayed(func_cached)(arg_single) for arg_single in arg)
    else:
        return func_cached(arg)

 

转载于:https://www.cnblogs.com/andy-0212/p/10314331.html