python实现分布式_python 分布式进程实现

import sys

import os

sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), os.path.pardir))

import Queue

import threading

import multiprocessing

from sfo_common.agent import Agent

from sfo_common.import_common import util, config, time, socket, logger, schedule

from multiprocessing.managers import BaseManager

# 定义一个任务队列

task_queue = Queue.Queue(1000)

# 定义一个类作为server，继承自BaseManager

class SQueueManager(BaseManager):

pass

# 定义一个类作为client,继承自BaseManager

class CQueueManager(BaseManager):

pass

def testfunc():

nowtime = time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))

logger.info("server:" + nowtime)

class TaskServer(Agent):

def __init__(self, pidfile):

Agent.__init__(self, pidfile)

def run(self):

sys.stdout.flush()

sys.stderr.flush()

hostname = socket.getfqdn()

hostip = socket.gethostbyname(hostname)

logger.info("the task server is running , its hostname is {}, ip is {}".format(hostname, hostip))

# 将队列注册到网络上

SQueueManager.register('get_task_queue', callable=lambda: task_queue)

# 将服务绑定到8102端口上，设置认证key为‘sfo_datahandler’

manager = SQueueManager(address=('0.0.0.0', 8102), authkey='sfo_datahandler')

# 启动Queue

manager.start()

try:

# 获得通过网络访问的Queue对象

task = manager.get_task_queue()

# 添加任务

def get_testfunc_schl():

task.put(testfunc)

# schdule

schedule.every(5).seconds.do(get_testfunc_schl)

schedule.run_all(0)

# os.wait()

while True:

schedule.run_pending()

time.sleep(0.1)

# 如果队列满了，说明非提供服务的主机，清空队列

if task.full():

task.clear()

except Exception as ex:

logger.exception(ex)

finally:

manager.shutdown()

class TaskClient(Agent):

def __init__(self, pidfile):

Agent.__init__(self, pidfile)

def run(self):

sys.stdout.flush()

sys.stderr.flush()

hostname = socket.getfqdn()

hostip = socket.gethostbyname(hostname)

logger.info("the task client is running , its hostname is {}, ip is {}".format(hostname, hostip))

# 由于这个queuemanager 只从网络上获取queue，所以注册时只提供名字

CQueueManager.register('get_task_queue')

# 连接到服务器，配置的ip为vip，避免单点故障

manager = CQueueManager(address=(config.task_server, 8102), authkey='sfo_datahandler')

# 从网络连接

manager.connect()

# 获取Queue的对象

tasks = manager.get_task_queue()

# 从task队列取任务

pool = Queue.Queue(config.process_workers)

while True:

logger.info("client:" + str(tasks.qsize()))

if tasks.empty():

time.sleep(10)

else:

if pool.full():

time.sleep(10)

else:

work = tasks.get(timeout=5)

pool.put(work)

logger.info("pool:" + str(pool.qsize()))

if not pool.empty():

wk = pool.get(1)

multiprocessing.Process(target=wk).start()

def main():

util.ensure_dir('/var/log/test/')

util.ensure_dir(config.sys_agent_pfile)

util.ensure_dir(config.agent_log_fname)

ts_pid = '/var/run/test/tsagent.pid'

tc_pid = '/var/run/test/tcagent.pid'

tsagent = TaskServer(ts_pid)

tcagent = TaskClient(tc_pid)

if len(sys.argv) == 2:

if 'start' == sys.argv[1]:

ts = threading.Thread(target=tsagent.start)

tc = threading.Thread(target=tcagent.start)

for t in [ts, tc]:

t.start()

time.sleep(0.1)

elif 'stop' == sys.argv[1]:

tcagent.stop()

tsagent.stop()

elif 'restart' == sys.argv[1]:

ts = threading.Thread(target=tsagent.restart)

tc = threading.Thread(target=tcagent.restart)

for t in [ts, tc]:

t.start()

time.sleep(0.1)

else:

raise IOError.message.format('Unknown command')

else:

print("usage: %s" % (sys.argv[1],))

sys.exit(2)

if __name__ == '__main__':

main()