kafka生产者消费者python版demo

1. 版本

Python 3.8.2

kafka-python 2.0.2

2. 生产者

import time
import json
from kafka import KafkaProducer

topic = 'sanford'
bootstrap_servers = 'localhost:9092'


def test_producer():
    producer = KafkaProducer(bootstrap_servers=bootstrap_servers,
                             key_serializer=lambda k: json.dumps(k).encode(),
                             value_serializer=lambda m: json.dumps(m).encode())

    data = {'name': 'jay', 'timestamp': int(time.time()*1000)}

    try:
        future = producer.send(topic=topic, value=data)
        future.get(timeout=10)
    except Exception as e:
        print('生产者发送失败:{0}'.format(e))
    else:
        print('生产者发送成功')


if __name__ == '__main__':
    test_producer()

3. 消费者

auto_offset_reset说明: earliest从未消费的开始消费; latest从最新的开始消费,在已提交offset情况下，lastest和earliest消费机制一致。未提交时，latest会丢失数据

import json
from kafka import KafkaConsumer

topic = 'sanford'
group_id = 'demo_python'
bootstrap_servers = 'localhost:9092'
# auto_offset_reset = 'earliest'
auto_offset_reset = 'latest'


def test_consumer():
    try:
        consumer = KafkaConsumer(
            topic,
            group_id=group_id,
            bootstrap_servers=bootstrap_servers,
            auto_offset_reset=auto_offset_reset)
        for msg in consumer:
            print('data:{0}'.format(json.loads(msg.value)))
            consumer.commit()
    except Exception as e:
        print('消费者接收失败:{0}'.format(e))


if __name__ == '__main__':
    test_consumer()