kafka-python
from kafka import KafkaConsumer,TopicPartition,OffsetAndMetadata
consumer = KafkaConsumer("wd-prd-data-test",
group_id="wd_prd_info_group",
auto_offset_reset='latest',
enable_auto_commit=True,
bootstrap_servers=['10.7.128.7:9092','10.7.128.8:9092','10.7.128.9:9092'])
tp = TopicPartition("wd-prd-data-test",0)
pp = [tp]
#partition的最旧(最小)有效offset
ef = consumer.beginning_offsets(pp)
#partition的最新(最大)offset
lf = consumer.end_offsets(pp)
#获取consumer group 对应的当前offset 对应partition消费过的最大offset
consumer.assignment()
cf = consumer.position(tp)
print(ef)
print(cf)
print(lf)
#dd = OffsetAndMetadata(418400,None)
#consumer.commit({tp:dd})
n=0
for message in consumer:
n=n+1
print (n," %s:%d:%d: key=%s value=%s" % (message.topic, message.partition,
message.offset, message.key,
message.value))
pykafka
from pykafka import KafkaClient
client = KafkaClient(hosts = '10.7.128.7:9092,10.7.128.8:9092,10.7.128.9:9092')
#client = KafkaClient(zookeeper_hosts = 'test1:2181,test5:2181,test6:2181/chroot/finai')
offset=0
topic = client.topics["wd-prd-data-test".encode()]
partitions = topic.partitions
last_offset = topic.latest_available_offsets()
print("最新最大offset {}".format(sorted(last_offset.items(),key=lambda item:item[0]))) # 查看所有分区
earliest_offset = topic.earliest_available_offsets()
print("最小(最旧)可用offset {}".format(sorted(earliest_offset.items(),key=lambda item:item[0]))) # 查看所有分区
"""
consumer = topic.get_balanced_consumer(b"simple_consumer_group",auto_commit_enable=True,
auto_commit_interval_ms=1,
zookeeper_connect = 'test1:2181,test5:2181,test6:2181/chroot/finai'
) # 选择一个分区进行消费
"""
consumer = topic.get_simple_consumer(b"simple_consumer_group",
auto_commit_enable=True,
auto_commit_interval_ms=1,
auto_offset_reset='latest' #必须参数,group一次消费时起作用
) # 选择一个分区进行消费
offset_list = consumer.held_offsets
# 消费者拥有的分区offset的情况,从0开始计数,对应partition已经消费过的最大offset-1
print("当前消费者分区offset情况{}".format(sorted(offset_list.items(),key=lambda item:item[0])))
#consumer.reset_offsets([(partitions[0], offset)]) # 设置offset
#msg = consumer.consume()
#print("消费 :{}".format(msg.value.decode()))
#offset = consumer.held_offsets
#print("当前消费者分区offset情况{}".format(offset)) # 3
n=0
for message in consumer:
if message is not None:
n=n+1
print(n," :: ",message.offset, message.value)