1. 多进程库multiprocessing常见方法
- p.start(),当前进程准备就绪,等待被CPU调度(工作单元其实是进程中的线程)
- p.join(),等待当前进程的任务执行完毕后再继续向下执行
- p.daemon = 布尔值,守护进程(必须放在start之前)
- p.daemon = True,设置为守护进程,主进程执行完毕后子进程也自动关闭
- p.daemon = False,设置为非守护进程,主进程等待子进程,子进程执行完毕后,主进程才结束
import time from multiprocessing import Process def task(arg): time.sleep(2) print('执行中...') if __name__ == '__main__': p1 = Process(target=task, args=('xxx',)) p1.start() p1.join() # p1结束再继续向下 p2 = Process(target=task, args=('xxx',)) p2.daemon = True # 设置p2为守护进程 p2.start()
- 进程名称的设置和获取
import time import multiprocessing def task(arg): time.sleep(2) print('当前进程的名称:%s', multiprocessing.current_process().name) if __name__ == '__main__': p = multiprocessing.Process(target=task, args=('xxx', )) p.name = '哈哈哈哈哈' p.start()
- 自定义进程类,直接将线程需要做的事写到run方法中
import multiprocessing class MyProcess(multiprocessing.Process): def run(self): print('执行此进程', self._args) if __name__ == '__main__': p = MyProcess(args=('xxx', )) p.start()
2. 进程数据共享
进程是资源分配的最小单元,每个进程中都维护自己独立的数据,不共享
实现进程间数据共享方法:
- Value
Value类型对应:from multiprocessing import Process, Value def f(n): n.value = 3.1415927 if __name__ == '__main__': num = Value('d', 0.0) p = Process(target = f, args=(num,)) p.start() p.join() print(num.value)
'c': ctypes.c_char, 'u': ctypes.c_wchar, 'b': ctypes.c_byte, 'B': ctypes.c_ubyte, 'h': ctypes.c_short, 'H': ctypes.c_ushort, 'i': ctypes.c_int, 'I': ctypes.c_uint, 'l': ctypes.c_long, 'L': ctypes.c_ulong, 'f': ctypes.c_float, 'd': ctypes.c_double
- Manager
import multiprocessing def task(data): for i in range(10): data.append(i) print('子进程中处理后的结果:', data) if __name__ == '__main__': manager = multiprocessing.Manager() data_list = manager.list() p = multoprocessing.Process(target=task, args=(data_list, )) p.start() p.join() print('主进程:', data_list)
- multiprocessing.Queue
import multiprocessing def task(q): for i in range(10): q.put(i) if __name__ == '__main__': queue = multiprocessing.Queue() p = multiprocessing.Process(target=task, args=(queue,)) p.start() p.join() print(queue.get()) print(queue.get()) print(queue.get())
- multiprocessing.Pipe
import time import multiprocessing def task(conn): time.sleep(1) conn.send([111, 22, 33, 44]) data = conn.recv() # 阻塞 print('子进程接收: ', data) time.sleep(2) if __name__ == '__main__': parent_conn, child_conn = multiprocessing.Pipe() p = multiprocessing.Process(target=task, args=(child_conn, )) p.start() info = parent_conn.recv() # 阻塞 print('主进程接收: ', info) parent_conn.send(666)
- 第三方工具也可以实现资源共享,如:reids、mysql
3. 进程锁
- 如果多个进程抢占式去做某些操作,为防止操作出现问题,可通过进程锁避免
import time import multiprocessing def task(lock): lock.acquire() with open('f1.txt', mode='r', encoding='utf-8') as f: current_num = int(f.read()) print('排队抢票了') time.sleep(1) current_num -= 1 with open('f1.txt', mode='w', encoding='utf-8') as f: f.write(str(current_num)) lock.release() if __name__ == '__main__': # multiprocessing.set_start_method('fork') # windows支持'spawn' Linux支持'fork' for i in range(10): p = multiprocessing.Process(target=task, args=(lock, )) p.start() time.sleep(10) # windows系统是spawn模式创建的进程,需要特殊处理(windows系统)。也可通过join等待
4. 进程池
import time
from concurrent.futures.process import ProcessPoolExecutor
import multiprocessing
def task(num):
time.sleep(2)
return num
def done(res):
print(multiprocessing.current_process())
time.sleep(1)
print(res.result())
time.sleep(1)
if __name__ == '__main__':
pool = ProcessingPoolExecutor(4)
for i in range(50):
fur = pool.submit(task, i)
fur.add_done_callback(done) # done的调用由主进程处理(与线程池不同)
print(multiprocessing.current_process())
pool.shutdown(True) # 等待进程池中的进程执行完毕
- 如果在进程池中要使用进程锁,则必须使用Manager中的Lock和RLock,而不能使用multiprocessing.RLock
import time import multiprocessing from concurrent.futures.process import ProcessPoolExecutor def task(lock): with lock: with open('f1.txt', mode='r', encoding='utf-8') as f: current_num = int(f.read()) print('排队抢票了') time.sleep(1) current_num -=1 with open('f1.txt', mode='w', encoding='utf-8') as f: f.write(str(current_num)) if __name__ == '__main__': pool = ProcessPoolExecutor() # lock_object = multiprocessing.RLock() # 不能使用!!! m = multiprocessing.Manager() lock_object = m.RLock() for i in range(10): pool.submit(task, lock_object)
5. 操作系统层面进程间通信方法
- 管道
- 消息队列
- 共享内存
- 信号量
- 信号
- socket