python多进程

单参数多线程

python多线程中的join，apply_async，apply，close，terminate
apply_async是非阻塞的，apply是阻塞的具体区别见下边例子，非阻塞跟c++的detach类似。

apply_async

import multiprocessing
import time
def func(msg):
    print "msg:", msg
    time.sleep(3)
    print "end"
    return "done" + msg

if __name__ == "__main__":
    pool = multiprocessing.Pool(processes=4)
    result = []
    for i in xrange(3):
        msg = "hello %d" %(i)
        result.append(pool.apply_async(func, (msg, )))
    pool.close()
    pool.join()
    for res in result:
        print ":::", res.get()
    print "Sub-process(es) done."

result如下，可以看到进程间是同步运行的没有等待

msg: hello 0
msg: hello 1
msg: hello 2
end
end
end
::: donehello 0
::: donehello 1
::: donehello 2
Sub-process(es) done.

apply

#coding: utf-8
import multiprocessing
import time

def func(msg):
    print "msg:", msg
    time.sleep(3)
    print "end"

if __name__ == "__main__":
    pool = multiprocessing.Pool(processes = 3)
    for i in xrange(4):
        msg = "hello %d" %(i)
        pool.apply(func, (msg, ))   #维持执行的进程总数为processes，当一个进程执行完毕后会添加新的进程进去

    print "Mark~ Mark~ Mark~~~~~~~~~~~~~~~~~~~~~~"
    pool.close()
    pool.join()   #调用join之前，先调用close函数，否则会出错。执行完close后不会有新的进程加入到pool,join函数等待所有子进程结束
    print "Sub-process(es) done."

result，可以看到进程之间是有等待关系的

msg: hello 0
end
msg: hello 1
end
msg: hello 2
end
msg: hello 3
end
Mark~ Mark~ Mark~~~~~~~~~~~~~~~~~~~~~~
Sub-process(es) done.

多参数多进程

对应上文的单线程事例

from multiprocessing import Pool

def my_print(x):
    print(x)
 
if __name__ == "__main__":
    x = [1, 2, 3, 4, 5]
    pool = Pool()
    pool.map(my_print, x)
    pool.close()
    pool.join()

多参数使用偏函数partial固定值不变的参数

from functools import partial
from multiprocessing import Pool
def my_print(x, y):
    print(x + y)
if __name__ == '__main__':# 多线程，多参数，partial版本
    x = [1, 2, 3, 4, 5]
    y = 1
    partial_func = partial(my_print, y=y)
    pool = Pool()
    pool.map(partial_func, x)
    pool.close()
    pool.join()

使用starmap函数，可以使用可变参数

if __name__ == '__main__': # 多线程，多参数，starmap版本
    x = [1, 2, 3, 4, 5]
    y = [1, 1, 1,3, 7]
    zip_args = list(zip(x, y))
    pool = Pool()
    pool.starmap(my_print, zip_args)
    pool.close()
    pool.join()