python多进程写同一个list/dict

python2下的写法

import time
from tqdm import tqdm
import multiprocessing as mp


def picklable_op(_class, *args):
    """
    多进程之间要使用pickle来序列化并传递一些数据。
    由于py2下实例方法并不能像py3一样直接被pickle。
    所以需要对多进程对象进行封装，使之可以在py2下被pickle。
    """
    return _class.proc_func(*args)


class OP():
    def __init__(self):
        # 直接调用 Manager 提供的 list() 和 dict()
        self.manager = mp.Manager
        self.mp_lst = self.manager().list()
        self.mp_dict = self.manager().dict()
        self.length = 64

    def proc_func(self, i, j):
        self.mp_lst.append(i)
        self.mp_dict[i] = j
        time.sleep(0.1)

    def flow(self):
        pool = mp.Pool(16)
        for i in range(self.length):
            pool.apply_async(picklable_op, args=(self, i, i*2))
        pool.close()
        pool.join()


if __name__ == '__main__':

    start_time = time.time()
    op = OP()
    op.flow()
    print(op.mp_lst)
    print(op.mp_dict)
    print(time.time() - start_time)

python3下的写法

import time
from tqdm import tqdm
import multiprocessing as mp


class OP():
    def __init__(self):
        # 直接调用 Manager 提供的 list() 和 dict()
        self.manager = mp.Manager
        self.mp_lst = self.manager().list()
        self.mp_dict = self.manager().dict()
        self.length = 64

    def proc_func(self, i, j):
        self.mp_lst.append(i)
        self.mp_dict[i] = j
        time.sleep(0.1)

    def flow(self):
        pool = mp.Pool(16)
        for i in range(self.length):
            pool.apply_async(self.proc_func, args=(i, i*2))
        pool.close()
        pool.join()


if __name__ == '__main__':

    start_time = time.time()
    op = OP()
    op.flow()
    print(op.mp_lst)
    print(op.mp_dict)
    print(time.time() - start_time)

---

参考文献

[1] 今天遇到的Python多线程、多进程中的几个坑