Python多进程

1. 多进程库multiprocessing常见方法

• p.start()，当前进程准备就绪，等待被CPU调度（工作单元其实是进程中的线程）
• p.join()，等待当前进程的任务执行完毕后再继续向下执行
• p.daemon = 布尔值，守护进程（必须放在start之前）
  • p.daemon = True，设置为守护进程，主进程执行完毕后子进程也自动关闭
  • p.daemon = False，设置为非守护进程，主进程等待子进程，子进程执行完毕后，主进程才结束

  import time
  from multiprocessing import Process
  
  def task(arg):
  	time.sleep(2)
  	print('执行中...')
  
  if __name__ == '__main__':
  	p1 = Process(target=task, args=('xxx',))
  	p1.start()
  	p1.join()  # p1结束再继续向下
  	p2 = Process(target=task, args=('xxx',))
  	p2.daemon = True  # 设置p2为守护进程
  	p2.start()
  

• 进程名称的设置和获取

  import time 
  import multiprocessing
  
  def task(arg):
  	time.sleep(2)
  	print('当前进程的名称：%s', multiprocessing.current_process().name)
  
  if __name__ == '__main__':
  	p = multiprocessing.Process(target=task, args=('xxx', ))
  	p.name = '哈哈哈哈哈'
  	p.start()
  

• 自定义进程类，直接将线程需要做的事写到run方法中

  import multiprocessing
  
  class MyProcess(multiprocessing.Process):
  	def run(self):
  		print('执行此进程', self._args)
  
  if __name__ == '__main__':
  	p = MyProcess(args=('xxx', ))
  	p.start()
  

2. 进程数据共享

进程是资源分配的最小单元，每个进程中都维护自己独立的数据，不共享
实现进程间数据共享方法：

• Value

  from multiprocessing import Process, Value
  
  def f(n):
  	n.value = 3.1415927
  
  if __name__ == '__main__':
  	num = Value('d', 0.0)
  	p = Process(target = f, args=(num,))
  	p.start()
  	p.join()
  	print(num.value)
  

  Value类型对应：

  		'c': ctypes.c_char,  'u': ctypes.c_wchar,
  	    'b': ctypes.c_byte,  'B': ctypes.c_ubyte,
  	    'h': ctypes.c_short, 'H': ctypes.c_ushort,
  	    'i': ctypes.c_int,   'I': ctypes.c_uint,
  	    'l': ctypes.c_long,  'L': ctypes.c_ulong,
  	    'f': ctypes.c_float, 'd': ctypes.c_double
  

• Manager

  import multiprocessing
  
  def task(data):
  	for i in range(10):
  		data.append(i)
  	print('子进程中处理后的结果：', data)
  
  if __name__ == '__main__':
  	manager = multiprocessing.Manager()
  	data_list = manager.list()
  	
  	p = multoprocessing.Process(target=task, args=(data_list, ))
  	p.start()
  	p.join()
  	print('主进程：', data_list)
  

• multiprocessing.Queue

  import multiprocessing
  
  def task(q):
  	for i in range(10):
  		q.put(i)
  
  if __name__ == '__main__':
  	queue = multiprocessing.Queue()
  	p = multiprocessing.Process(target=task, args=(queue,))
  	p.start()
  	p.join()
  	print(queue.get())
  	print(queue.get())
  	print(queue.get())
  

• multiprocessing.Pipe

  import time
  import multiprocessing
  
  def task(conn):
  	time.sleep(1)
  	conn.send([111, 22, 33, 44])
  	data = conn.recv()  # 阻塞
  	print('子进程接收: ', data)
  	time.sleep(2)
  
  if __name__ == '__main__':
  	parent_conn, child_conn = multiprocessing.Pipe()
  	p = multiprocessing.Process(target=task, args=(child_conn, ))
  	p.start()
  	info = parent_conn.recv()  # 阻塞
  	print('主进程接收: ', info)
  	parent_conn.send(666)
  

• 第三方工具也可以实现资源共享，如：reids、mysql

3. 进程锁

• 如果多个进程抢占式去做某些操作，为防止操作出现问题，可通过进程锁避免

  import time
  import multiprocessing
  
  def task(lock):
  	lock.acquire()
  	with open('f1.txt', mode='r', encoding='utf-8') as f:
  		current_num = int(f.read())
  
  	print('排队抢票了')
  	time.sleep(1)
  	current_num -= 1
  
  	with open('f1.txt', mode='w', encoding='utf-8') as f:
  		f.write(str(current_num))
  	lock.release()
  
  if __name__ == '__main__':
  	# multiprocessing.set_start_method('fork')  # windows支持'spawn'  Linux支持'fork'
  	for i in range(10):
  		p = multiprocessing.Process(target=task, args=(lock, ))
  		p.start()
  		
  	time.sleep(10)  # windows系统是spawn模式创建的进程，需要特殊处理（windows系统）。也可通过join等待
  

4. 进程池

import time
from concurrent.futures.process import ProcessPoolExecutor
import multiprocessing

def task(num):
	time.sleep(2)
	return num

def done(res):
	print(multiprocessing.current_process())
	time.sleep(1)
	print(res.result())
	time.sleep(1)

if __name__ == '__main__':
	pool = ProcessingPoolExecutor(4)
	for i in range(50):
		fur = pool.submit(task, i)
		fur.add_done_callback(done)  # done的调用由主进程处理（与线程池不同）
	
	print(multiprocessing.current_process())
	pool.shutdown(True)  # 等待进程池中的进程执行完毕

• 如果在进程池中要使用进程锁，则必须使用Manager中的Lock和RLock，而不能使用multiprocessing.RLock

  import time
  import multiprocessing
  from concurrent.futures.process import ProcessPoolExecutor
  
  def task(lock):
  	with lock:
  		with open('f1.txt', mode='r', encoding='utf-8') as f:
  			current_num = int(f.read())
  		
  		print('排队抢票了')
  		time.sleep(1)
  		current_num -=1
  		
  		with open('f1.txt', mode='w', encoding='utf-8') as f:
  			f.write(str(current_num))
  
  if __name__ == '__main__':
  	pool = ProcessPoolExecutor()
  	# lock_object = multiprocessing.RLock()  # 不能使用！！！
  	m = multiprocessing.Manager()
  	lock_object = m.RLock()
  	for i in range(10):
  		pool.submit(task, lock_object)
  

5. 操作系统层面进程间通信方法

• 管道
• 消息队列
• 共享内存
• 信号量
• 信号
• socket