from concurrent.futures import ProcessPoolExecutor
from functools import partial
from multiprocessing import cpu_count
from tqdm import tqdm
input = [] #some input
out_dir = 'output'
def run(args1, args2, ...):
pass
def multi_cpu_run(num_workers=cpu_count()):
executor = ProcessPoolExecutor(max_workers=num_workers)
futures = []
for one_input in input:
# some process
futures.append(executor.submit(
partial(run, args1, args2, ...)))
return [future.result() for future in tqdm(futures)]
if __name__ == '__main__':
result = multi_cpu_run()
with open(os.path.join(out_dir, 'output.txt'), 'w', encoding='utf-8') as f:
for m in result:
f.write(','.join([str(x) for x in m]) + '\n')
这个run函数的处理时间最好长一点,返回结果会写在output.txt中。