线程代替方案
- subprocess
- 完全跳过线程,使用进程
- 是派生进行的主要替代方案
- python2.4后进入
- multiprocessiong
- 使用thronging接口派生,使用子进程
- 允许为多核或者多cpu派生进程,接口跟threading非常相似
- python2.6
- concurrent.futures
- 新的异步执行模块
- 任务级别的操作
- python3.2后引入
多进程
- 进程间通讯(InterprocessCommunication,IPC)
- 进程之间无任何共享状态
- 进程的创建
- 直接生成Process实例对象。
- 派生子类。
# 直接生成Process实例对象
# -*- coding:utf-8 -*-
import multiprocessing
from time import sleep, ctime
def clock(interval):
i = 0
while i < 5:
print("The time is %s" % ctime())
sleep(interval)
i += 1
if __name__ == '__main__':
p = multiprocessing.Process(target=clock, args=(5,))
p.start()
The time is Mon Jul 2 14:07:34 2018
The time is Mon Jul 2 14:07:39 2018
The time is Mon Jul 2 14:07:44 2018
The time is Mon Jul 2 14:07:49 2018
The time is Mon Jul 2 14:07:54 2018
# 派生子类
# -*- coding:utf-8 -*-
import multiprocessing
from time import sleep, ctime
class ClockProcess(multiprocessing.Process):
"""
两个函数比较
1、init构造函数
2、run
"""
def __init__(self, interval):
super().__init__()
self.interval = interval
def run(self):
i = 0
while i < 5:
print("The time is %s" % ctime())
sleep(self.interval)
i += 1
if __name__ == "__main__":
p = ClockProcess(3)
p.start()
The time is Mon Jul 2 14:08:09 2018
The time is Mon Jul 2 14:08:12 2018
The time is Mon Jul 2 14:08:15 2018
The time is Mon Jul 2 14:08:18 2018
The time is Mon Jul 2 14:08:21 2018
- 在os中查看pid,ppid的关系
# -*- coding:utf-8 -*-
from multiprocessing import Process
import os
def info(title):
print(title)
print("module name:", __name__)
# 得到父进程的ID
print("parent process:", os.getppid())
# 得到本身进程的ID
print("process id :", os.getpid())
def f(name):
info('function f')
print('hello', name)
if __name__ == '__main__':
info('main line')
p = Process(target=f, args=('whj',))
p.start()
main line
module name: __main__
parent process: 19586
process id : 20428
function f
module name: __main__
parent process: 20428
process id : 20463
hello whj
生产者消费者模型
- JoinableQueue
#生产者、消费者多进程的实现
#JoinableQueue好处:可以带通知,可以等子进程都完成
import multiprocessing
from time import sleep, ctime
def consumer(input_q):
print("Into consumer:", ctime())
i = 0
while i < 100:
# 预处理项
item = input_q.get()
print("pull", item, "out of q") # 此项替换为有用的工作
input_q.task_done() # 发出信号通知任务完成
i += 1
print("Out of consumer:", ctime())
def producer(sequence, output_q):
print("Into producer:", ctime())
for item in sequence:
output_q.put(item)
print("put", item, "into q")
print("Out of producer:", ctime())
if __name__ == "__main__":
q = multiprocessing.JoinableQueue()
# 运行消费者进行
qc = multiprocessing.Process(target=consumer, args=(q,))
qc.daemon = True
qc.start()
# 生产多个项,sequence代表要发给消费者的序列项
# 在实践中,这可能是生成器的输出或通过一些其他方法的返回
sequence = [1, 2, 3, 4]
producer(sequence, q)
# 等待所有项被处理
#要等待,如果不等待,那么主进程完了,还有东西没有消费完
q.join()
- 对列中哨兵的使用:通知作用
# -*- coding:utf-8 -*-
import multiprocessing
from time import sleep, ctime
def consumer(input_q):
print("Into consumer:", ctime())
while True:
# 预处理项
item = input_q.get()
if item is None:
break
print("pull", item, "out of q") # 此项替换为有用的工作
input_q.task_done() # 发出信号通知任务完成
print("Out of consumer:", ctime())
def producer(sequence, output_q):
print("Into producer:", ctime())
for item in sequence:
output_q.put(item)
print("put", item, "into q")
print("Out of producer:", ctime())
if __name__ == "__main__":
q = multiprocessing.JoinableQueue()
# 运行消费者进行
qc = multiprocessing.Process(target=consumer, args=(q,))
qc.daemon = True
qc.start()
# 生产多个项,sequence代表要发给消费者的序列项
# 在实践中,这可能是生成器的输出或通过一些其他方法的返回
sequence = [1, 2, 3, 4, -1]
producer(sequence, q)
q.put(None) # 哨兵值
# 等待所有项被处理
qc.join()
- 哨兵的改进,假设有多个消费者,通知多个消费者让他们停止,就是多加了个none值
#哨兵的改进,假设有多个消费者,通知多个消费者让他们停止,就是多加了个none值
import multiprocessing
from time import sleep, ctime
def consumer(input_q):
print("Into consumer:", ctime())
while True:
# 预处理项
item = input_q.get()
if item is None:
break
print("pull", item, "out of q") # 此项替换为有用的工作
input_q.task_done() # 发出信号通知任务完成
print("Out of consumer:", ctime())
def producer(sequence, output_q):
print("Into producer:", ctime())
for item in sequence:
output_q.put(item)
print("put", item, "into q")
print("Out of producer:", ctime())
if __name__ == "__main__":
#Queue和JoinableQueue差不多
q = multiprocessing.Queue()
# 运行消费者进行
qc1 = multiprocessing.Process(target=consumer, args=(q,))
qc1.daemon = True
qc1.start()
qc2 = multiprocessing.Process(target=consumer, args=(q,))
qc2.daemon = True
qc2.start()
# 生产多个项,sequence代表要发给消费者的序列项
# 在实践中,这可能是生成器的输出或通过一些其他方法的返回
sequence = [1, 2, 3, 4, -1]
producer(sequence, q)
q.put(None) # 哨兵值
q.put(None)
# 等待所有项被处理
qc1.join()
qc2.join()