python2下的写法
import time
from tqdm import tqdm
import multiprocessing as mp
def picklable_op(_class, *args):
"""
多进程之间要使用pickle来序列化并传递一些数据。
由于py2下实例方法并不能像py3一样直接被pickle。
所以需要对多进程对象进行封装,使之可以在py2下被pickle。
"""
return _class.proc_func(*args)
class OP():
def __init__(self):
self.manager = mp.Manager
self.mp_lst = self.manager().list()
self.mp_dict = self.manager().dict()
self.length = 64
def proc_func(self, i, j):
self.mp_lst.append(i)
self.mp_dict[i] = j
time.sleep(0.1)
def flow(self):
pool = mp.Pool(16)
for i in range(self.length):
pool.apply_async(picklable_op, args=(self, i, i*2))
pool.close()
pool.join()
if __name__ == '__main__':
start_time = time.time()
op = OP()
op.flow()
print(op.mp_lst)
print(op.mp_dict)
print(time.time() - start_time)
python3下的写法
import time
from tqdm import tqdm
import multiprocessing as mp
class OP():
def __init__(self):
self.manager = mp.Manager
self.mp_lst = self.manager().list()
self.mp_dict = self.manager().dict()
self.length = 64
def proc_func(self, i, j):
self.mp_lst.append(i)
self.mp_dict[i] = j
time.sleep(0.1)
def flow(self):
pool = mp.Pool(16)
for i in range(self.length):
pool.apply_async(self.proc_func, args=(i, i*2))
pool.close()
pool.join()
if __name__ == '__main__':
start_time = time.time()
op = OP()
op.flow()
print(op.mp_lst)
print(op.mp_dict)
print(time.time() - start_time)
参考文献
[1] 今天遇到的Python多线程、多进程中的几个坑