python消费kafka某个topic某个分区的数据

不负长风

于 2024-06-05 16:50:50 发布

阅读量225

点赞数 1

分类专栏： # python后端开发文章标签： python

本文链接：https://blog.csdn.net/qq_32862515/article/details/139476607

版权

python后端开发专栏收录该内容

12 篇文章 0 订阅

订阅专栏

from confluent_kafka import Consumer, TopicPartition

kafka_ip_port = 'ip:port'
consumer = Consumer({
        'bootstrap.servers': kafka_ip_port,
        'group.id': 'test_group',
        'auto.offset.reset': 'earliest'
})

topics = consumer.list_topics().topics

# print(topics, type(topics))
# topic_name = list(topics.keys())[0]

topic_name = 'your_topic_name'

topic_meta = topics[topic_name]

# partitions = list(topic_meta.partitions.keys())
# partition = partitions[0]

partition = 2

print(f'prepare to consume topic: {topic_name}, partition: {partition}')

consumer.assign([TopicPartition(topic=topic_name, partition=partition, offset=0)])

messages = consumer.consume(num_messages=5, timeout=5)

for message in messages:
    if message.error():
        print(f"consume error: {message.error()}")
        continue
    print(f'offset: {message.offset()}, key: {message.key().decode()}, value: {message.value().decode()}')