kafka代码实战

一、CLI

# 创建topic名称为first，3个分区，1个副本
./kafka-topics.sh --zookeeper 192.168.233.129:12181 --create --topic first --replication-factor 1 --partitions 3
# 查看first此topic信息
./kafka-topics.sh --zookeeper 192.168.233.129:12181 --describe --topic first
Topic: first        PartitionCount: 3        ReplicationFactor: 1        Configs: 
        Topic: first        Partition: 0        Leader: 2        Replicas: 2        Isr: 2
        Topic: first        Partition: 1        Leader: 0        Replicas: 0        Isr: 0
        Topic: first        Partition: 2        Leader: 1        Replicas: 1        Isr: 1
# 调用生产者生产消息
./kafka-console-producer.sh --broker-list 192.168.233.129:19092,192.168.233.129:19093,192.168.233.129:19094 --topic first
# 调用消费者消费消息，from-beginning表示读取全部的消息
./kafka-console-consumer.sh --bootstrap-server 192.168.233.129:19092,192.168.233.129:19093,192.168.233.129:19094 --topic first --from-beginning
# 删除topic
./kafka-topic.sh --zookeeper 192.168.233.129:12181 --delete --topic second

二、API

2.2 生产者端（同步、异步、回调）

from kafka import KafkaProducer
from kafka.errors import KafkaError

producer = KafkaProducer(bootstrap_servers=['broker1:1234'])

# 默认为异步方式
future = producer.send('my-topic', b'raw_bytes')

# 定义阻塞以同步方式发送
try:
    record_metadata = future.get(timeout=10)
except KafkaError:
    # 如果生产请求失败
    log.exception()
    pass

# 成功，返回分配的分区和偏移量
print (record_metadata.topic)
print (record_metadata.partition)
print (record_metadata.offset)

# 生成分区键，将消息发布到分区
producer.send('my-topic', key=b'foo', value=b'bar')

# 通过msgpack编码对象
producer = KafkaProducer(value_serializer=msgpack.dumps)
producer.send('msgpack-topic', {'key': 'value'})

# 生成json消息
producer = KafkaProducer(value_serializer=lambda m: json.dumps(m).encode('ascii'))
producer.send('json-topic', {'key': 'value'})

# 产生异步
for _ in range(100):
    producer.send('my-topic', b'msg')

def on_send_success(record_metadata):
    print(record_metadata.topic)
    print(record_metadata.partition)
    print(record_metadata.offset)

def on_send_error(excp):
    log.error('I am an errback', exc_info=excp)
    # 处理异常

# 使用回调异步生成
producer.send('my-topic', b'raw_bytes').add_callback(on_send_success).add_errback(on_send_error)

# 阻塞，直到发送所有异步消息
producer.flush()

# 配置多个重试
producer = KafkaProducer(retries=5)

2.2 单个消费者（offset、定时任务）

from kafka import KafkaConsumer

consumer = KafkaConsumer('test',bootstrap_servers=['127.0.0.1:9092'])  #参数为接收主题和kafka服务器地址

# 这是一个永久堵塞的过程，生产者消息会缓存在消息队列中,并且不删除,所以每个消息在消息队列中都有偏移
for message in consumer:  # consumer是一个消息队列，当后台有消息时，这个消息队列就会自动增加．所以遍历也总是会有数据，当消息队列中没有数据时，就会堵塞等待消息带来
    print("%s:%d:%d: key=%s value=%s" % (message.topic, message.partition,message.offset, message.key,message.value))

2.3 消费者组

from kafka import KafkaConsumer
# 使用group,对于同一个group的成员只有一个消费者实例可以读取数据
consumer = KafkaConsumer('test',group_id='my-group',bootstrap_servers=['127.0.0.1:9092'])
for message in consumer:
    print("%s:%d:%d: key=%s value=%s" % (message.topic, message.partition,message.offset, message.key,message.value))