mysql自动重置重置t_kafka将offset自动重置为最新

1.可以将kafka中的偏移量自动重置为最新的

2.使用于kafka有积压,但是也不想处理积压,直接消费最新的数据

3.此版本只支持offset存储在zk中, 暂未提供offset存储在kafka中的版本

# -*- coding:utf-8 -*-

import time

import sys

from kafka.client_async import KafkaClient

from kafka.protocol.commit import OffsetCommitRequest_v0

from kafka.protocol.offset import OffsetRequest_v0, OffsetResponse_v0

from kafka.structs import TopicPartition

servers = '192.168.0.1:9092,192.168.0.2:9092,192.168.0.3:9092'

gid = 'group_id'

topic = 'topic'

# 手动重置时需要配置,同时修改main方法中调用方法

manual_logsize = sys.maxint

def parse_logsize(t, p, responses):

"""

单个broker中单个partition的logsize

:param responses:

:param p:

:param t:

:return:

"""

for response in responses:

if not isinstance(response, OffsetResponse_v0):

continue

tps = response.topics

tpc = tps[0][0]

partition_list = tps[0][1]

parti = partition_list[0][0]

if tpc == t and parti == p and partition_list[0][1] == 0:

logsize_list = partition_list[0][2]

logsize = logsize_list[0]

return logsize

return None

def auto_lateset(g, t):

"""

自动重置为最新的offset

"""

client = KafkaClient(bootstrap_servers=servers, request_timeout_ms=3000)

partitions = client.cluster.partitions_for_topic(t)

for partition in partitions:

nodeId = client.cluster.leader_for_partition(TopicPartition(topic=t, partition=partition))

while not client.is_ready(nodeId):

client.ready(nodeId)

time.sleep(1)

client.send(nodeId, OffsetRequest_v0(replica_id=-1, topics=[(t, [(partition, -1, 1)])]))

log_size = parse_logsize(t, partition, client.poll(timeout_ms=3000))

if log_size:

client.send(nodeId, OffsetCommitRequest_v0(consumer_group=g,

topics=[(t, [(partition, log_size, '')])]))

print client.poll()

def manual(g, t, log_size):

"""

手动重置offset为 manual_logsize的值,注意:所有分区都会重置

"""

client = KafkaClient(bootstrap_servers=servers, request_timeout_ms=3000)

partitions = client.cluster.partitions_for_topic(t)

for partition in partitions:

nodeId = client.cluster.leader_for_partition(TopicPartition(topic=t, partition=partition))

while not client.is_ready(nodeId):

client.ready(nodeId)

time.sleep(1)

client.send(nodeId, OffsetCommitRequest_v0(consumer_group=g,

topics=[(t, [(partition, log_size, '')])]))

print client.poll()

if __name__ == "__main__":

auto_lateset(gid, topic)

# manual(gid, topic, manual_logsize)