concurrent.futures 中的chunksize

import time
from multiprocessing.pool import Pool
from concurrent.futures import as_completed, ProcessPoolExecutor

NUMBERS = range(1, 100000)
K = 50


def f(x):
    r = 0
    for k in range(1, K + 2):
        r += x ** (1 / k ** 1.5)
    return r


if __name__ == '__main__':

    start = time.time()

    l = []
    pool = Pool(3)
    for num, result in zip(NUMBERS, pool.map(f, NUMBERS)):
        l.append(result)
    print(len(l))
    print('cost:{}'.format(time.time() - start))


    print('ProcessPoolExecutor without chunksize:\n')
    start = time.time()
    l = []
    with ProcessPoolExecutor(max_workers=3) as executor:
        for num, result in zip(NUMBERS, executor.map(f, NUMBERS)):
            l.append(result)
    print(len(l))
    print('cost:{}'.format(time.time() - start))


    print('ProcessPoolExecutor with chunksize:\n')
    start = time.time()
    l = []
    with ProcessPoolExecutor(max_workers=3) as executor:
        chunksize, extra = divmod(len(NUMBERS), executor._max_workers * 4)
        for num, result in zip(NUMBERS, executor.map(f, NUMBERS, chunksize=chunksize)):
            l.append(result)
    print(len(l))

    print('COST: {}'.format(time.time() - start))

运行结果：

multiprocessing.pool

99999
cost:0.23070788383483887

ProcessPoolExecutor without chunksize:

99999
cost:7.456393241882324

ProcessPoolExecutor with chunksize:

99999
COST: 0.2285900115966797

Process finished with exit code 0