Python 多线程消费Kafka 提高吞吐量

from kafka import KafkaConsumer, KafkaAdminClient, TopicPartition
from multiprocessing import Process
import sys
 
 
def run(topic_name, partition_id, group_id):
    consumer = KafkaConsumer(bootstrap_servers=bootstrap_servers,
                             max_poll_records=500,  # 每次poll最大的记录数
                             max_poll_interval_ms=30000,  # 两次间隔poll最大间隔时间ms
                             heartbeat_interval_ms=10000,  # 每次心跳间隔ms
                             session_timeout_ms=20000,  # 心跳会话超时时间ms
                             group_id=group_id
                             )
    # 为进程指定消费的topic名称、分区id
    consumer.assign([TopicPartition(topic_name, partition_id)])
    while True:
        messages = consumer.poll(timeout_ms=1000)  # 拉取每次超时时间, 如果1000ms内拉到500条记录则直接返回,如果超过了1000ms拉不到500条也返回
        for tp, messages_list in messages.items():
            for message in messages_list:
                print(message.value.decode(), message.offset)
 
 
if __name__ == '__main__':
 
    # 获取分区数
    bootstrap_servers = ['localhost:9092']
    adminClient = KafkaAdminClient(bootstrap_servers=bootstrap_servers)
    topic = 'test'
    topics = ['test']
    group_id = 'python-consumer'
    tps = adminClient.describe_topics()
    partitions_len = len(tps[0]['partitions'])
 
    if partitions_len <= 0:
        print("topic 分区数 <= 0")
        sys.exit(1)
 
    # 开启多进程, 进程数 = 分区数
    processes = []
    for i in range(partitions_len):
        p = Process(target=run, args=(topic, i, group_id,))
        p.start()
        processes.append(p)
 
    # 等待所有进程结束
    for p in processes:
        p.join()