python 日志输出到kafka_如何从Python日志模块写入Kafka？

太棒了，谢谢！在过去的几年中，代码有了一些更新，现在有些函数已被弃用。这个修补程序的总体设计非常非常有帮助，所以再次感谢您。SimpleProducer (deprecated) --> KafkaProducer

SimpleConsumer (deprecated) --> KafkaConsumer

这是我使用Kafka 1.0.0和kafkapython1.4.2的修订片段，只是制作者，因为我正在另一端通过logstash消费。

希望这对你有用！

tester.py(主程序)# -*- coding: utf-8 -*-

"""Module to test out logging to kafka."""

import json

import logging

from utils.kafka_handler import KafkaHandler

from kafka import KafkaProducer

def run_it(logger=None):

"""Run the actual connections."""

logger = logging.getLogger(__name__)

# enable the debug logger if you want to see ALL of the lines

#logging.basicConfig(level=logging.DEBUG)

logger.setLevel(logging.DEBUG)

kh = KafkaHandler(['localhost:9092'], 'sebtest')

logger.addHandler(kh)

logger.info("I'm a little logger, short and stout")

logger.debug("Don't tase me bro!")

if __name__ == "__main__":

run_it()

utils/kafka_handler.py(日志实用程序)# -*- coding: utf-8 -*-

"""Module to provide kafka handlers for internal logging facility."""

import json

import logging

import sys

from kafka import KafkaProducer

class KafkaHandler(logging.Handler):

"""Class to instantiate the kafka logging facility."""

def __init__(self, hostlist, topic='corp_it_testing', tls=None):

"""Initialize an instance of the kafka handler."""

logging.Handler.__init__(self)

self.producer = KafkaProducer(bootstrap_servers=hostlist,

value_serializer=lambda v: json.dumps(v).encode('utf-8'),

linger_ms=10)

self.topic = topic

def emit(self, record):

"""Emit the provided record to the kafka_client producer."""

# drop kafka logging to avoid infinite recursion

if 'kafka.' in record.name:

return

try:

# apply the logger formatter

msg = self.format(record)

self.producer.send(self.topic, {'message': msg})

self.flush(timeout=1.0)

except Exception:

logging.Handler.handleError(self, record)

def flush(self, timeout=None):

"""Flush the objects."""

self.producer.flush(timeout=timeout)

def close(self):

"""Close the producer and clean up."""

self.acquire()

try:

if self.producer:

self.producer.close()

logging.Handler.close(self)

finally:

self.release()