kafka_2.10-0.8.X 以及之前的版本

1 适用于 kafka_2.10-0.8.X 以及之前的版本

该版本代码本人试验通过，kafka版本为：kafka_2.10-0.8.2.1

2 使用依赖

<dependency>
      <groupId>org.apache.kafka</groupId>
      <artifactId>kafka_2.10</artifactId>
      <version>0.10.0.1</version>
 </dependency>

3 代码

  public class KafkaOffsetMonitor_0_8_2_1 {

    private static final Logger logger = LoggerFactory.getLogger(KafkaOffsetMonitor_0_8_2_1.class);

    public static void main(String[] args) {

        String brokerStr = args[0];//不带端口号的kafka服务地址，如：192.168.0.1,192.168.0.2
        String topic = args[1];//主题名
        String group = args[2];//消费组名称
        logger.info("brokerStr" + ":" + brokerStr);
        logger.info("topic" + ":" + topic);
        logger.info("group" + ":" + group);
        
        List<String> brokers = Arrays.asList(brokerStr.split(","));
        int port = 9092;

        String clientId = "Client-" + topic;
        int correlationId = 0;

        KafkaOffsetMonitor_0_8_2_1 monitor = new KafkaOffsetMonitor_0_8_2_1();
        Map<Integer, PartitionMetadata> partitionMetadataMap = monitor.getPartitoinsMetadata(brokers, port, topic);

        List<TopicAndPartition> topicAndPartitions = new ArrayList<>();
        for (Map.Entry<Integer, PartitionMetadata> entry : partitionMetadataMap.entrySet()) {
            TopicAndPartition topicAndPartition = new TopicAndPartition(topic, entry.getKey());
            topicAndPartitions.add(topicAndPartition);
        }

        Map<TopicAndPartition, OffsetMetadataAndError> offsets = monitor.getCommittedOffset(brokers.get(0), port, group, topicAndPartitions, correlationId, clientId);

        long logSize = 0L;
        long sumOffset = 0L;
        long lag = 0L;

        for (Map.Entry<Integer, PartitionMetadata> entry : partitionMetadataMap.entrySet()) {

            Integer partitionId = entry.getKey();
            String host = entry.getValue().leader().host();
            String clientName = "Client-" + topic + "-" + partitionId;

            TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partitionId);
            OffsetMetadataAndError committedOffsetMetadata = offsets.get(topicAndPartition);

            if (committedOffsetMetadata.error() != ErrorMapping.NotCoordinatorForConsumerCode()) {
                long committedOffset = committedOffsetMetadata.offset();
                sumOffset += committedOffset;
            }

            // kafka包括两个常数
            // kafka.api.OffsetRequest.EarliestTime()发现日志中的数据的开始
            // kafka.api.OffsetRequest.LatestTime()将只留新消息
            long partitionOffset = monitor.getPartitionOffset(host, port, topic, partitionId, OffsetRequest.LatestTime(), clientName);
            logSize += partitionOffset;
        }

        lag = logSize - sumOffset;
        logger.info("====================kafka=========================");
        logger.info("topic:" + topic);
        logger.info("logSize:" + logSize);
        logger.info("sumOffset:" + sumOffset);
        logger.info("lag:" + lag);
        logger.info("==================================================");
    }


    /**
     * 根据 partitionId 获取topic指定分区的偏移量（总偏移量）
     * 使用 low level api
     *
     * @param host        分区leader ip
     * @param port        端口号
     * @param topic       主题名
     * @param partitionId 分区id
     * @param whichTime   最晚时间
     * @param clientName  客户端名称
     * @return
     */
    public long getPartitionOffset(String host, int port, String topic, int partitionId, long whichTime, String clientName) {
        // 创建连接
        SimpleConsumer simpleConsumer = new SimpleConsumer(host, port, 100000, 64 * 1024, clientName);

        // 组装请求
        TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partitionId);
        Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<>();
        requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));

        kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName);
        // 发送请求
        OffsetResponse response = simpleConsumer.getOffsetsBefore(request);

        /**
         * SimpleConsumer 不处理broker错误，有错误需要自己处理
         */
        if (response.hasError()) {

            logger.error("Error fetching data Offset Data the Broker. Reason: " + response.errorCode(topic, partitionId));
            return 0L;
        }

        long[] offsets = response.offsets(topic, partitionId);

        if (simpleConsumer != null) {
            simpleConsumer.close();
        }

        return offsets[0];
    }


    /**
     * 获取 topic 下的所有已提交偏移量
     * （备注：0.8及以下版本可以适用，0.9及以上的返回的offsets为负数）
     * 使用 low level api
     *
     * @param broker             kafka服务ip
     * @param port               端口
     * @param group              消费组
     * @param topicAndPartitions topicAndPartitions组
     * @param correlationId      关联id (由客户端指定的一个数字唯一标示这次请求的id，服务器端在处理完请求后也会把同样的CorrelationId写到Response中，这样客户端就能把某个请求和响应对应起来了)
     * @param clientId           客户端id
     * @return
     */
    public Map<TopicAndPartition, OffsetMetadataAndError> getCommittedOffset(String broker, int port, String group, List<TopicAndPartition> topicAndPartitions, int correlationId, String clientId) {

        // 创建连接
        BlockingChannel channel = new BlockingChannel(broker, port, BlockingChannel.UseDefaultBufferSize(), BlockingChannel.UseDefaultBufferSize(), 5000);
        channel.connect();

        // 组装请求
        OffsetFetchRequest offsetFetchRequest = new OffsetFetchRequest(group, topicAndPartitions,
                (short) 0,  // by default bind to version 0 so that it fetches from ZooKeeper
                correlationId,
                clientId);

        // 发送请求
        channel.send(offsetFetchRequest.underlying());
        // 获取返回数据
        OffsetFetchResponse fetchResponse = OffsetFetchResponse.readFrom(channel.receive().payload());
        Map<TopicAndPartition, OffsetMetadataAndError> offsets = fetchResponse.offsets();

        if (channel.isConnected()) {
            channel.disconnect();
        }

        return offsets;
    }


    /**
     * 获取 topic 的分区信息
     * 使用 low level api
     *
     * @param brokers kafka服务器列表ip
     * @param prot    端口号
     * @param topic   主题名
     * @return
     */
    public Map<Integer, PartitionMetadata> getPartitoinsMetadata(List<String> brokers, int prot, String topic) {

        // 使用 map 对 PartitionMetadata 进行去重
        Map<Integer, PartitionMetadata> result = new HashMap<>();

        for (String broker : brokers) {
            // 创建连接
            SimpleConsumer simpleConsumer = new SimpleConsumer(broker, prot, 100000, 64 * 1024, "Client-PartitoinsMetadata-" + broker);
            // 组装请求
            List<String> topics = Collections.singletonList(topic);
            TopicMetadataRequest request = new TopicMetadataRequest(topics);
            // 发送请求
            TopicMetadataResponse response = simpleConsumer.send(request);
            // 获取返回数据
            for (TopicMetadata topicsMetadatum : response.topicsMetadata()) {
                for (PartitionMetadata partitionsMetadatum : topicsMetadatum.partitionsMetadata()) {
                    result.put(partitionsMetadatum.partitionId(), partitionsMetadatum);
                }
            }

            if (simpleConsumer != null) {
                simpleConsumer.close();
            }
        }

        return result;
    }
}