Kafka java端获取Topoic，新建Topic，offset（当前偏移量，最终偏移量）信息

Kafka java端获取Topoic，新建Topic，offset（当前偏移量，最终偏移量）信息

网上很多介绍怎么获取偏移量的，但是都是互抄，而且版本过老，方法不适用，或者要创建Consumer才能获取偏移量，我就奇了怪了，监听自动创建Consumer，新建再获取什么鬼，所以这里介绍的是使用Kafka自带的Api获取。

一些解释都写在了注释里

依赖

implementation group: 'org.springframework.kafka', name: 'spring-kafka', version: '2.6.6'

配置文件//仅供参考

#kafka
spring.kafka.bootstrap-servers=？？？？
#消费者数量，不大于分片数
spring.kafka.listener.concurrency=10
spring.kafka.producer.value-serializer=org.springframework.kafka.support.serializer.JsonSerializer
spring.kafka.consumer.value-deserializer=org.springframework.kafka.support.serializer.JsonDeserializer
#消费者反序列化信任后才行
spring.kafka.consumer.properties.spring.json.trusted.packages=*
spring.kafka.consumer.group-id=test

创建AdminClient Bean对象

import org.apache.kafka.clients.admin.AdminClient;
import org.apache.kafka.clients.admin.AdminClientConfig;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.kafka.core.KafkaAdmin;

import java.util.HashMap;
import java.util.Map;

/**
 * @author xvjinjiang
 * create at  2021/6/1 15:01
 */
@Configuration
public class KafkaConfig {

    @Value("${spring.kafka.bootstrap-servers}")
    private String bootstrapServers;

    @Bean //创建一个kafka管理类，相当于rabbitMQ的管理类rabbitAdmin,没有此bean无法自定义的使用adminClient创建topic
    public KafkaAdmin kafkaAdmin() {
        Map<String, Object> props = new HashMap<>();
        //配置Kafka实例的连接地址
        //kafka的地址，不是zookeeper
        props.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
        KafkaAdmin admin = new KafkaAdmin(props);
        return admin;
    }

    @Bean  //kafka客户端，在spring中创建这个bean之后可以注入并且创建topic,用于集群环境，创建对个副本
    public AdminClient adminClient() {
        return AdminClient.create(kafkaAdmin().getConfig());
    }

}

接口实现

import io.swagger.v3.oas.annotations.Operation;
import lombok.extern.slf4j.Slf4j;
import org.apache.kafka.clients.admin.*;
import org.apache.kafka.clients.consumer.OffsetAndMetadata;
import org.apache.kafka.common.KafkaFuture;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.TopicPartitionInfo;
import org.springframework.kafka.config.KafkaListenerContainerFactory;
import org.springframework.web.bind.annotation.*;

import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;

/**
 * @author xvjinjiang
 * create at  2021/6/1 15:08
 */
@RestController
@RequestMapping("kafka")
@Slf4j
public class KafkaController {

    private final AdminClient adminClient;

    public KafkaController(AdminClient adminClient) {
        this.adminClient = adminClient;
    }

    @PostMapping("create-topic")
    @Operation(operationId = "createTopic", summary = "创建新kafkaTopic")
    public void createTopic(@RequestParam String topicName, @RequestParam Integer num) {
        //主题名称、分区数和副本数
        NewTopic topic = new NewTopic(topicName, num, (short) 1);
        adminClient.createTopics(Collections.singletonList(topic));
    }

    @GetMapping("topic-names")
    @Operation(operationId = "getTopics", summary = "获取Topics")
    public Set<String> getTopics() {
        ListTopicsResult listTopics = adminClient.listTopics();
        try {
            return listTopics.names().get();
        } catch (Exception e) {
            log.error(e.getMessage());
        }
        return null;
    }

    /**
     * 获取topic下partition信息
     */
    @GetMapping("topic-describe")
    @Operation(operationId = "getTopicsInfo", summary = "获取Topics")
    public List<String> getTopicsInfo(@RequestParam String topicName) throws Exception {
        DescribeTopicsResult describeTopicsResult = adminClient.describeTopics(Collections.singleton(topicName));
        KafkaFuture<Map<String, TopicDescription>> all = describeTopicsResult.all();
        Map<String, TopicDescription> stringTopicDescriptionMap = all.get();
        TopicDescription topicDescription = stringTopicDescriptionMap.get(topicName);
        List<String> res = topicDescription.partitions().stream().map(TopicPartitionInfo::toString).collect(Collectors.toList());
        return res;

    }

    /**
     * 获取groupId下的偏移信息
     */
    @GetMapping("group-describe")
    @Operation(operationId = "getGroupInfo", summary = "获取GroupInf")
    public  List<KafkaGroupInfo> getGroupInfo(@RequestParam String groupId,@RequestParam String topic) throws Exception {

        ListConsumerGroupOffsetsResult listConsumerGroupOffsetsResult = adminClient.listConsumerGroupOffsets(groupId);
        //当前偏移量
        Map<TopicPartition, OffsetAndMetadata> topicPartitionOffsetAndMetadataMap = listConsumerGroupOffsetsResult.partitionsToOffsetAndMetadata().get(10, TimeUnit.SECONDS);
        Map<TopicPartition, OffsetSpec> map = new HashMap<>();
        topicPartitionOffsetAndMetadataMap.keySet().forEach(x -> {
            map.put(x, OffsetSpec.latest());
        });
        ListOffsetsResult listOffsetsResult = adminClient.listOffsets(map);
        Set<TopicPartition> topicPartitions = topicPartitionOffsetAndMetadataMap.keySet();
        List<KafkaGroupInfo> res = new ArrayList<>();
        for (TopicPartition topicPartition : topicPartitions) {
            if (!topicPartition.toString().startsWith(topic))continue;
            //最终偏移量
            ListOffsetsResult.ListOffsetsResultInfo listOffsetsResultInfo = listOffsetsResult.partitionResult(topicPartition).get();
            //拼装返回对象
            KafkaGroupInfo kafkaGroupInfo = new KafkaGroupInfo();
            //toString=》topic + "-" + partition
            kafkaGroupInfo.consumerName = topicPartition.toString();
            Long lastOffset = listOffsetsResultInfo.offset();
            kafkaGroupInfo.lastOffset = lastOffset;
            Long currentOffset = topicPartitionOffsetAndMetadataMap.get(topicPartition).offset();
            kafkaGroupInfo.currentOffset = currentOffset;
            kafkaGroupInfo.lag = lastOffset - currentOffset;
            res.add(kafkaGroupInfo);
        }
        return res;
    }
}