Python 多线程编写 Kafka Producer

多线程生产到同一 topic

#!/usr/bin/env python
# -*- coding: utf-8 -*-
import random
import time
import threading
from kafka import KafkaProducer
from traitlets import log


def on_send_success(record_metadata):
    print(record_metadata.topic)
    print(record_metadata.partition)
    print(record_metadata.offset)


def on_send_error(excp):
    log.error('I am an errback', exc_info=excp)
    # handle exception

def send_messag(topic, id, interval_time):

    producer = KafkaProducer(bootstrap_servers=['cdh01:9092', 'cdh02:9092', 'cdh03:9092'])
    while True:
        startTime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())

        message = "|".join((id, startTime, str(interval_time)))

        producer.send(topic, message.encode("utf-8")).add_callback(on_send_success).add_errback(on_send_error)
        # 间隔多长时间生产一条消息
        time.sleep(interval_time)

interval_times = [60, 300, 600, 1200, 1800, 3600]

#开启50个线程
for i in range(0, 50):
    deviceId = "%06d" % random.randint(0, 999999)

    interval_time = times[random.randint(0, len(interval_times) - 1)]

    recv_thread = threading.Thread(target=send_messag, args=("my-topic", id, interval_time))
    
    # 守护主线程，主线程退出后子线程直接销毁
    recv_thread.setDaemon(True)
    recv_thread.start()

while True:
    #主线程一直运行
    time.sleep(5000)