Python并发编程—多进程multiprocess(2)
进程通信和数据共享
Pipe
multiprocessing.Pipe([duplex])
用于两个进程之间的连接,返回表示管道末端的 Connection 对象的对 (conn1, conn2)。
参数duplex默认为True,则管道是双工的,两边都能发送和结束,duplex为False,则管道是单向的,conn1只能用于接收,conn2只能用于发送
import multiprocessing
from multiprocessing import Pipe
def process1(conn1):
print(multiprocessing.current_process())
print(conn1.recv())
if __name__ == "__main__":
conn1, conn2 = Pipe()
pro1 = multiprocessing.Process(target=process1, args=(conn1,))
pro1.start()
conn2.send('hello world')
<Process(Process-1, started)>
hello world
队列
multiprocess有三个队列对象Queue,SimpleQueue及JoinableQueue
— Queue
qsize():返回队列大小,不可靠
empty():返回队列是否空,不可靠
full():返回队列是否已满,不可靠
put(obj[, block[, timeout]]): 将obj放入队列中,可选参数block默认True,timeout默认None,则此时会阻塞直到队列空闲 可用,设置timeout,最多阻塞timeout秒,超时抛出queue.Full异常
put_nowait(obj): 相当于put(obj,False)
get([block[, timeout]])
get_nowait()
— SimpleQueue
queue的简化版
— JoinableQueue
queue的子类,多了task_done()和join()方法
task_done() :由队列的使用者调用,get()获取的任务执行完,调用task_done()通知任务完成
join(): 阻塞直到队列任务全部完成
import multiprocessing
def worker(jobs, results):
while True:
a = jobs.get()
a **= 2
results.put(a)
jobs.task_done()
def create_process(jobs, results):
for _ in range(4):
process = multiprocessing.Process(target=worker, args=(jobs, results))
process.daemon = True
process.start()
def add_jobs(jobs):
for i in range(20):
jobs.put(i)
def start():
jobs = multiprocessing.JoinableQueue() # 任务队列,会阻塞直到任务完成
results = multiprocessing.Queue() # 结果对列
create_process(jobs, results) # 创建进程并启动,处于阻塞状态,加入任务之后开始执行
add_jobs(jobs) # 添加任务
jobs.join()
while not results.empty(): # empty()方法此时是可靠的,任务已执行完成
result = results.get_nowait() # 获取结果,打印
print(result)
if __name__ == "__main__":
start()
通过内存共享数据
multiprocessing.Value(typecode_or_type, *args, lock=True)
返回从共享内存分配的 ctypes 对象
multiprocessing.Array(typecode_or_type, size_or_initializer, *, lock=True)
返回从共享内存分配的ctypes数组
from multiprocessing import Process, Value, Array
def process1(a, b):
a.value += 1
for index, i in enumerate(b):
b[index] = i ** 2
def process2(a, b):
a.value *= 3
for index, i in enumerate(b):
b[index] = i + 2
if __name__ == "__main__":
a = Value('i', 4)
b = Array('i', [1, 2, 3])
pro1 = Process(target=process1, args=(a, b))
pro2 = Process(target=process2, args=(a, b))
pro1.start()
pro2.start()
pro1.join()
pro2.join()
print(a.value)
print([i for i in b])
15
[3, 6, 11]
通过网络或内存共享数据
— BaseManager
multiprocessing.managers.BaseManager([address[, authkey]])
start([initializer[, initargs]])
get_server()
connect()
shutdown()
register(typeid[, callable[, proxytype[, exposed[, method_to_typeid[, create_method]]]]])
address:只读属性
from multiprocessing.managers import SyncManager
from multiprocessing import Process
from time import sleep
class MathsClass:
def add(self, x, y):
return x + y
def mul(self, x, y):
return x * y
class MathManager(SyncManager):
pass
def process1(m_list, m):
sleep(1)
print(m.add(m_list[0], m_list[8]))
def process2(m_list):
for i in range(1, 10):
m_list.append(i)
if __name__ == "__main__":
MathManager.register('Maths', MathsClass)
with MathManager() as m:
y = m.Maths()
x = m.list()
pro1 = Process(target=process1, args=(x, y))
pro2 = Process(target=process2, args=(x,))
pro1.start()
pro2.start()
pro1.join()
pro2.join()