java监控kafka获取 logSize offset lag

窦了豆子

已于 2023-04-14 14:18:32 修改

阅读量631

点赞数

文章标签： java kafka

于 2023-04-13 18:26:12 首次发布

本文链接：https://blog.csdn.net/weixin_41131698/article/details/130137013

版权

<!-- kafka 监控Api  排除slf4j-log4j12  防止报错 V    注意和spring-kafka的版本兼容-->
        <dependency>
            <groupId>org.apache.kafka</groupId>
            <artifactId>kafka-clients</artifactId>
            <version>3.4.0</version>
            <exclusions>
                <exclusion>
                    <groupId>org.slf4j</groupId>
                    <artifactId>slf4j-log4j12</artifactId>
                </exclusion>
            </exclusions>
        </dependency>

在这里插入图片描述

package com.orientalsemi.oceanplus.dataetl.util;

import lombok.extern.slf4j.Slf4j;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.clients.consumer.OffsetAndMetadata;
import org.apache.kafka.common.PartitionInfo;
import org.apache.kafka.common.TopicPartition;

import java.util.*;

@Slf4j
public class KafkaUtil {

    public static long KafkaConsumeLag(String server, String groupId, String topic){
        Map endOffsetMap = new HashMap();
        Map commitOffsetMap = new HashMap();
        Properties consumeProps = getConsumeProperties(groupId, server);
        log.info("consumer properties:" + consumeProps);
        //查询topic partitions
        KafkaConsumer consumer = new KafkaConsumer(consumeProps);
        List topicPartitions = new ArrayList();
        List partitionsFor = consumer.partitionsFor(topic);
        for (Object map : partitionsFor) {
            PartitionInfo partitionInfo = (PartitionInfo) map;
            TopicPartition topicPartition = new TopicPartition(partitionInfo.topic(), partitionInfo.partition());
            topicPartitions.add(topicPartition);
        }
        //查询logsize
        Map endOffsets = consumer.endOffsets(topicPartitions);
        for (Object map : endOffsets.keySet()) {
            TopicPartition partitionInfo = (TopicPartition) map;
            endOffsetMap.put(partitionInfo.partition(), endOffsets.get(partitionInfo));
        }
        for (Object partitionId : endOffsetMap.keySet()) {
            log.info(String.format("at %s, topic:%s, partition:%s, logSize:%s", System.currentTimeMillis(), topic, partitionId, endOffsetMap.get(partitionId)));
        }
        //查询消费offset
        for (Object map : topicPartitions) {
            TopicPartition topicAndPartition = (TopicPartition) map;
            OffsetAndMetadata committed = consumer.committed(topicAndPartition);
            commitOffsetMap.put(topicAndPartition.partition(), committed.offset());
        }
        //累加lag
        long lagSum = 0l;
        if (endOffsetMap.size() == commitOffsetMap.size()) {
            for (Object data : endOffsetMap.keySet()) {
                long endOffSet = (long) endOffsetMap.get(data);
                long commitOffSet = (long) commitOffsetMap.get(data);
                long diffOffset = endOffSet - commitOffSet;
                lagSum += diffOffset;
                log.info("Topic:" + topic + ", groupID:" + groupId + ", partition:" + data + ", endOffset:" + endOffSet + ", commitOffset:" + commitOffSet + ", diffOffset:" + diffOffset);
            }
            log.info("Topic:" + topic + ", groupID:" + groupId + ", LAG:" + lagSum);
        } else {
            log.info("this topic partitions lost");
        }
        consumer.close();
        return lagSum;
    }



    public static Properties getConsumeProperties(String groupID, String bootstrap_server) {
        Properties props = new Properties();
        props.put("group.id", groupID);
        props.put("bootstrap.servers", bootstrap_server);
        props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        return props;
    }
}