python用kafka做消息队列

最新推荐文章于 2025-03-31 20:07:17 发布

南星叨叨

最新推荐文章于 2025-03-31 20:07:17 发布

阅读量2.1k

点赞数

分类专栏： python 文章标签： python kafka

本文链接：https://blog.csdn.net/hans99812345/article/details/123332540

版权

python 专栏收录该内容

83 篇文章

订阅专栏

你得搭建 kafka zk 这样这个东西才能跑

生产者代码

#!/usr/bin/python
# encoding=utf-8

import datetime
import json

from common.OperationTimeHelper import OperationTimeHelper
from common.LoggerHelper import LogggerHelper
from kafka import KafkaProducer

operation = OperationTimeHelper()

today = datetime.date.today()
log_name = "logs/%s_postfix_antispam.log" % str(today)
logger = LogggerHelper('debug').my_log(log_name)




def start_log(log):
    with open(log, 'r', encoding='utf-8') as file_object:
        file_object.seek(0, 2)
        while True:
            # last_pos = file_object.tell()
            line = file_object.readline()
            if line:
                new_line = line.strip()
             
                    mail_json = parse_log(new_line)
                    conf = {
                        'bootstrap_servers': ["kafka-zookeeper-01:9092"],
                        'topic_name': 'send_antispam',
                    }
                    # print('start producer')
                    producer = KafkaProducer(bootstrap_servers=conf['bootstrap_servers'],value_serializer=lambda v: json.dumps(v).encode('utf-8'))
                    # data = bytes("hello kafka!", encoding="utf-8")
                    producer.send(conf['topic_name'], mail_json)
                    # producer.close()
                    # print('end producer')


if __name__ == '__main__':
    log_file = '/var/log/mail.log'
    start_log(log_file)

消费者代码

import json

from kafka import KafkaConsumer

conf = {
    'bootstrap_servers': ["kafka-zookeeper-01:9092"],
    'topic_name': 'send_antispam'
    # 'consumer_id': 'consumer-id'
}

print('start consumer')
consumer = KafkaConsumer(conf['topic_name'],
                         bootstrap_servers=conf['bootstrap_servers'])
# group_id=conf['consumer_id'])


# for message in consumer: print("%s:%d:%d: key=%s value=%s" % (message.topic, message.partition, message.offset,
# message.key, message.value))

for message in consumer:
    data = json.loads(message.value.decode('utf-8'))
    print(data)

print('end consumer')