Python的multiprocessing
模块不但支持多进程,其中managers
子模块还支持把多进程分布到多台机器上。一个服务进程可以作为调度者,将任务分布到其他多个进程中,依靠网络通信。由于managers
模块封装很好,不必了解网络通信的细节,就可以很容易地编写分布式多进程程序。
class Job:
def __init__(self, job_id):
self.job_id = job_id
from multiprocessing import Queue
from multiprocessing.managers import BaseManager
from PythonLearning.processjob import Job
class Master:
def __init__(self):
# 派发出去的作业队列
self.dispatched_job_queue = Queue()
# 完成的作业队列
self.finished_job_queue = Queue()
def get_dispatched_job_queue(self):
return self.dispatched_job_queue
def get_finished_job_queue(self):
return self.finished_job_queue
def start(self):
# 把派发作业队列和完成作业队列注册到网络上
BaseManager.register('get_dispatched_job_queue', callable=self.get_dispatched_job_queue)
BaseManager.register('get_finished_job_queue', callable=self.get_finished_job_queue)
# 监听端口和启动服务
manager = BaseManager(address=('127.0.0.1', 8888), authkey=b'jobs')
manager.start()
# 使用上面注册的方法获取队列
dispatched_jobs = manager.get_dispatched_job_queue()
finished_jobs = manager.get_finished_job_queue()
# 这里一次派发10个作业,等到10个作业都运行完后,继续再派发10个作业
job_id = 0
# while True:
for i in range(0, 20):
job_id = job_id + 1
job = Job(job_id)
print('Dispatch job: %s' % job.job_id)
dispatched_jobs.put(job)
while not dispatched_jobs.empty():
job = finished_jobs.get(60)
print('Finished Job: %s' % job.job_id)
manager.shutdown()
if __name__ == "__main__":
master = Master()
master.start()
import time
from multiprocessing.managers import BaseManager
from PythonLearning.processjob import Job
class Slave:
# def __init__(self):
# # 派发出去的作业队列
# self.dispatched_job_queue = Queue()
# # 完成的作业队列
# self.finished_job_queue = Queue()
def start(self):
# 把派发作业队列和完成作业队列注册到网络上
BaseManager.register('get_dispatched_job_queue')
BaseManager.register('get_finished_job_queue')
# 连接master
server = '127.0.0.1'
print('Connect to server %s...' % server)
manager = BaseManager(address=(server, 8888), authkey=b'jobs')
manager.connect()
# 使用上面注册的方法获取队列
dispatched_jobs = manager.get_dispatched_job_queue()
finished_jobs = manager.get_finished_job_queue()
# 运行作业并返回结果,这里只是模拟作业运行,所以返回的是接收到的作业
while True:
job = dispatched_jobs.get(timeout=1)
print('Run job: %s ' % job.job_id)
time.sleep(1)
result_job_id=job.job_id**2
result_job = Job(result_job_id)
finished_jobs.put(result_job)
if __name__ == "__main__":
slave = Slave()
slave.start()
"C:\Program Files (x86)\Python36-32\python.exe" D:/python_work/PythonLearning/processmanager.py
Dispatch job: 1
Dispatch job: 2
Dispatch job: 3
Dispatch job: 4
Dispatch job: 5
Dispatch job: 6
Dispatch job: 7
Dispatch job: 8
Dispatch job: 9
Dispatch job: 10
Dispatch job: 11
Dispatch job: 12
Dispatch job: 13
Dispatch job: 14
Dispatch job: 15
Dispatch job: 16
Dispatch job: 17
Dispatch job: 18
Dispatch job: 19
Dispatch job: 20
Finished Job: 1
Finished Job: 4
Finished Job: 9
Finished Job: 16
Finished Job: 25
Finished Job: 36
Finished Job: 49
Finished Job: 64
Finished Job: 81
Finished Job: 100
Finished Job: 121
Finished Job: 144
Finished Job: 169
Finished Job: 196
Finished Job: 225
Finished Job: 256
Finished Job: 289
Finished Job: 324
Finished Job: 361
Process finished with exit code 0