package xxx.util; import java.util.ArrayList; import java.util.Collection; import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; import org.springframework.stereotype.Component; import kafka.javaapi.*; import kafka.api.PartitionOffsetRequestInfo; import kafka.cluster.Broker; import kafka.common.TopicAndPartition; import kafka.javaapi.consumer.SimpleConsumer; import lombok.extern.slf4j.Slf4j; /** * Kafka工具类 * */ @Component @Slf4j public class KafkaUtil {
/** * for SimpleConsumer */ final int TIMEOUT = 100000 ; /** * for SimpleConsumer */ final int BUFFERSIZE = 64 * 1024 ; /** * 获取Kafka各topic最新offset * @param brokerList * @param topics * @param groupId * @return key: 分区, value: offset */ public Map<TopicAndPartition, Long> getLatestOffset(String brokerList, Collection<String> topics, String groupId) {
Map<TopicAndPartition, Long> partitionOffsetMap = new HashMap<TopicAndPartition, Long>(); List<String> topicList = new ArrayList<String>(); topicList.addAll(topics); Map<TopicAndPartition, Broker> partitionBrokerMap = findLeader(brokerList, topicList); for (Map.Entry<TopicAndPartition, Broker> partitionBrokerEntry : partitionBrokerMap .entrySet()) {
// get leader broker Broker leaderBroker = partitionBrokerEntry.getValue(); SimpleConsumer simpleConsumer = new SimpleConsumer(leaderBroker.host(), leaderBroker.port(), TIMEOUT, BUFFERSIZE, groupId); long readOffset = getTopicAndPartitionLatestOffset(simpl
|