python queue 多进程_基于queue的python多进程日志管理

在我们的异常检测应用中，需要对每组IoT设备分别训练一个模型，每个模型对一组设备的指标数据进行实时异常检测。方案采用master-worker的方式实现模型对外服务，但是每个worker的日志需要集中收集到同一个日志文件，而不是每个worker产生一个日志文件。此时我们采用基于logging模块的QueueHandler+QueueListener方案。master在创建worker之前，首先创建一个日志队列，创建worker时，将日志队列传入，同时worker端使用QueueHandler作为日志处理例程，将日志写入队列，master端启动Queuelistener对日志队列中的消息进行监听，对接收到的消息调用相应的FileHandler处理例程进行处理，写入文件。以次实现对多worker日志的集中管理。相关代码如下：

master侧的日志相关代码：

def_init_logger(self):

self.logger= logging.getLogger("job.manager")

self.log_queue=Queue()

self.log_queue_listener= QueueListener(self.log_queue, *self.logger.handlers)def_register_signals(self):

signal.signal(signal.SIGINT, self.stop)

signal.signal(signal.SIGTERM, self.stop)def_start_single_worker(self, worker_id, worker_group, worker_config):

worker=Worker(

worker_id=worker_id,

worker_group=worker_group,

group_members=worker_config.get('group_members'),

consume_topic=mq_config.get('sample_topic'),

consume_subscription=worker_group + '_subscription',

produce_topic=mq_config.get('result_topic'),

model_path=worker_config.get('model').get('binary_path'),

model_type=worker_config.get('model').get('type'),

log_queue=self.log_queue)try:

worker.start()exceptException as e:raise Exception("worker start error")

worker侧日志相关代码：

def __init__(self, worker_id, worker_group, group_members, consume_topic, consume_subscription,

produce_topic, model_path, model_type, log_queue):

self.id=worker_id

self.group=worker_group,

self.group_members=group_members

self.running=True

self.status= 'idle'self.logger=None

self.mq_client=None

self.model=None

self._init_logger(log_queue)

self._init_mq_client(consume_topic, consume_subscription, produce_topic)

self._load_model(model_path, model_type)

self._register_signals()

self.logger.info('Worker {} started.'.format(self.id))def __del__(self):ifself.mq_client:

self.mq_client.close()

@propertydefstatus(self):

with self._mutex:returnself._status

@status.setterdefstatus(self, status):

with self._mutex:

self._status=statusdef_init_logger(self, log_queue):

queue_handler=QueueHandler(log_queue)

self.logger= logging.getLogger('job.worker')

self.logger.addHandler(queue_handler)def_register_signals(self):

signal.signal(signal.SIGINT, self.stop)

signal.signal(signal.SIGTERM, self.stop)