Python连接Kafka

生产者

import json
from kafka import KafkaProducer, KafkaConsumer

producer = KafkaProducer(bootstrap_servers=['192.168.40.129:9092'])
msg_dict = {
    "msg": "Hello World"
}
msg = json.dumps(msg_dict).encode('utf-8')
producer.send('test', msg, partition=0)
print('success send')
producer.close()

异步发送，返回一个future对象

future = producer.send('test', msg, partition=0)
print(future)
print(future.get())


<kafka.producer.future.FutureRecordMetadata object at 0x00000265200E1438>
RecordMetadata(topic='test', partition=0, topic_partition=TopicPartition(topic='test', partition=0), offset=14, timestamp=1556267646972, checksum=None, serialized_key_size=-1, serialized_value_size=145, serialized_header_size=-1)

添加callback

def send_back(send_future):
    print('success')
    print(send_future)

def err_back(err_future):
    print('error')
    print(err_future)

producer.send('test',msg).add_callback(send_back).add_errback(err_back)

success
RecordMetadata(topic='test', partition=0, topic_partition=TopicPartition(topic='test', partition=0), offset=16, timestamp=1556268625293, checksum=None, serialized_key_size=-1, serialized_value_size=145, serialized_header_size=-1)

消费者

1. 普通消费者
   加入一个使用者组以执行动态分区分配和偏移量提交

import json
from kafka import  KafkaConsumer

consumer = KafkaConsumer('test',bootstrap_servers='192.168.40.129:9092')
print('consumer success')
print(consumer.topics()) # 打印所有topic
for msg in consumer:
    recv = "%s:%d:%d: key=%s value=%s" % (msg.topic, msg.partition, msg.offset, msg.key, msg.value)
    print(recv)

2. 加入指定消费者组

consumer = KafkaConsumer('test',bootstrap_servers='192.168.40.129:9092',group_id = 'my_favorite_group')
for msg in consumer:
    recv = "%s:%d:%d: key=%s value=%s" % (msg.topic, msg.partition, msg.offset, msg.key, msg.value)
    print(recv)

3. 手动分配分区partition与offset

consumer = KafkaConsumer(bootstrap_servers='192.168.40.129:9092')
consumer.assign([TopicPartition('test',0)])
print(consumer.assignment())
# consumer.assign([TopicPartition('test', 1)])
consumer.seek(TopicPartition(topic='test',partition=0),offset=10) # 手动分配offset
for msg in consumer:
    recv = "%s:%d:%d: key=%s value=%s" % (msg.topic, msg.partition, msg.offset, msg.key, msg.value)
    print(recv)

封装类

import json
from kafka import KafkaProducer, KafkaConsumer


class KTools(object):
    def __init__(self):
        self.producer = self.get_producer()

    @staticmethod
    def get_producer():
        """
        连接至kafka集群
        :return: producer(输入的value需要为dict)
        """
        producer = KafkaProducer(value_serializer=lambda v: json.dumps(v).encode('utf-8'),
                                 bootstrap_servers=['192.168.40.129:9092'
                                                    ])
        return producer

    def get_consumer(self, topic=None, offset='latest'):
        """
        连接至kafka集群
        :return: consumer
        """
        consumer = KafkaConsumer(topic,
                                 auto_offset_reset=offset,
                                 value_deserializer=lambda v: json.loads(
                                     v.decode('utf-8')),
                                 bootstrap_servers=['192.168.40.129:9092'
                                                    ])
        return consumer

    def get_topices(self):
        consumer = KafkaConsumer(bootstrap_servers=['192.168.40.129:9092'
                                                    ])
        return consumer.topics()


    def flush(self):
        self.producer.flush()


if __name__ == '__main__':
    test = KTools()
    consumer = test.get_consumer('py_test')
    for message in consumer:
        print("%s:%d:%d: key=%s value=%s" % (
            message.topic, message.partition, message.offset, message.key, message.value))
    future = test.producer.send('py_test', key=b'test', value={
                                'URL': 'www.example.com'})
    test.producer.flush()