解决python进程池中无法使用multiprocessing.Queue

使用python进程库中的进程池Pool可以简便的对进程任务进行管理,
同时python的multiprocessing为进程间通讯提供了不少工具,
其中之一便是消息队列Queue.
在测试时,将Queue作为一个参数传递到新进程时出现了问题,
传递Queue会导致进程启动失败.

import multiprocessing as MP
import datetime as DT
import os

def main(x, y, q):
    print('%s  %s' % (str(DT.datetime.today()), str(x*y)))
    q.put(x*y)

if __name__ == '__main__':
    q = MP.Queue()

    with MP.Pool(os.cpu_count()) as pool:
        for i in range(0,25):
            #这里如果将q置为None,则进程能成功启动
            rtn = pool.apply_async(main, (i, i+1, q))
        pool.close()
        pool.join()

    print('finish')

    while True:
        pass

而如果使用Process启动新进程则无问题,以下是官方文档的例子

from multiprocessing import Process, Queue

def f(q):
    q.put([42, None, 'hello'])

if __name__ == '__main__':
    q = Queue()
    p = Process(target=f, args=(q,))
    p.start()
    print(q.get())    # prints "[42, None, 'hello']"
    p.join()

解决这个问题的方案有两种
1.在Pool中初始化Queue

import multiprocessing as MP

def f(msg):  
    f.q.put(msg)   

def q_init(q):  
    f.q = q   

if __name__ == "__main__":  
    q = MP.Queue()  
    p = MP.Pool(None, q_init, (q,))
    for i in range(0,8):
        p.apply_async(f, (i,))
    p.close()
    p.join()

    while not q.empty():  
        print(q.get())

2.使用multiprocessing.Manager管理临界区

import multiprocessing as MP
import datetime as DT
import os

def main(x, y, ls):
    print('%s  %s' % (str(DT.datetime.today()), str(x*y)))
    ls.append(x*y)

if __name__ == '__main__':
    mgr = MP.Manager()
    p = mgr.list()

    with MP.Pool(os.cpu_count()) as pool:
        for i in range(0,25):
            rtn = pool.apply_async(main, (i, i+1, p))
        pool.close()
        pool.join()

    print('finish')

    print(p)

    while True:
        pass