python multiprocess

import multiprocessing
import numpy as np

data_pairs = [ [3,5], [4,3], [7,3], [1,6] ]

# define what to do with each data pair ( p=[3,5] ), example: calculate product
def myfunc(p):
    product_of_list = np.prod(p)
    return product_of_list

if __name__ == '__main__':
    pool = multiprocessing.Pool(processes=4)
    result_list = pool.map(myfunc, data_pairs)
    print(result_list)

偏函数

import multiprocessing
from functools import partial

data_list = [1, 2, 3, 4]

def prod_xy(x,y):
    return x * y

def parallel_runs(data_list):
    pool = multiprocessing.Pool(processes=4)
    prod_x=partial(prod_xy, y=10) # prod_x has only one argument x (y is fixed to 10)
    result_list = pool.map(prod_x, data_list)
    print(result_list)

if __name__ == '__main__':
    parallel_runs(data_list)

[10, 20, 30, 40]

starmap

# Parallelizing with Pool.starmap()
import multiprocessing as mp

def howmany_within_range(row, minimum, maximum):
    """Returns how many numbers lie within `maximum` and `minimum` in a given `row`"""
    count = 0
    for n in row:
        if minimum <= n <= maximum:
            count = count + 1
    return count

import numpy as np
from time import time

# Prepare data
np.random.RandomState(100)
arr = np.random.randint(0, 10, size=[200000, 5])
data = arr.tolist()
data[:5]

results = []
for row in data:
    results.append(howmany_within_range(row, minimum=4, maximum=8))

print(results[:10])
#> [3, 1, 4, 4, 4, 2, 1, 1, 3, 3]

pool = mp.Pool(mp.cpu_count())


results = pool.starmap(howmany_within_range, [(row, 4, 8) for row in data])

pool.close()

print(results)
print(len(results))
#> [3, 1, 4, 4, 4, 2, 1, 1, 3, 3]