Springboot-整合kafka

Spring for Apache Kafka 版本对应关系：

Spring for Apache Kafka Version	Spring Integration for Apache Kafka Version	kafka-clients	Spring Boot
2.7.0-SNAPSHOT	5.4.x	2.7.0 (RC5)	2.4.x
2.6.x	5.3.x or 5.4.x	2.6.0	2.3.x or 2.4.x
2.5.x	3.3.x	2.5.1	2.3.x
2.4.x	3.2.x	2.4.1	2.2.x
2.3.x	3.2.x	2.3.1	2.2.x
2.2.x	3.1.x	2.0.1, 2.1.x, 2.2.x	2.1.x (End of Life)
2.1.x	3.0.x	1.0.2	2.0.x (End of Life)
1.3.x	2.3.x	0.11.0.x, 1.0.x	1.5.x (End of Life)

官网链接：Spring for Apache Kafka

一、pom文件

        <dependency>
            <groupId>org.springframework.kafka</groupId>
            <artifactId>spring-kafka</artifactId>
        </dependency>

二、配置文件

server:
  port: 9000
# kafka
spring:
  kafka:
    #============== kafka ===================
    # 指定kafka server的地址，集群配多个，中间，逗号隔开
    bootstrap-servers: localhost:9092
    #=============== consumer  =======================
    consumer:
      # 指定默认消费者group id
      group-id: meng
      # earliest：当各分区下有已提交的offset时，从提交的offset开始消费；无提交的offset时，从头开始消费
      # latest：当各分区下有已提交的offset时，从提交的offset开始消费；无提交的offset时，消费新产生的该分区下的数据
      # none：topic各分区都存在已提交的offset时，从offset后开始消费；只要有一个分区不存在已提交的offset，则抛出异常
      auto-offset-reset: earliest
      # 指定消息key和消息体的编解码方式
      key-deserializer: org.apache.kafka.common.serialization.StringDeserializer
      value-deserializer: org.apache.kafka.common.serialization.StringDeserializer
    #=============== provider  =======================
    producer:
      # 指定消息key和消息体的编解码方式
      key-serializer: org.apache.kafka.common.serialization.StringSerializer
      value-serializer: org.apache.kafka.common.serialization.StringSerializer
      # 每次批量发送消息的数量
      batch-size: 65536
      buffer-memory: 524288
      bootstrap-servers: localhost:9092

三、代码

import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.SpringApplication;
import org.springframework.kafka.annotation.KafkaListener;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.kafka.support.SendResult;
import org.springframework.util.concurrent.ListenableFuture;
import org.springframework.util.concurrent.ListenableFutureCallback;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;

import java.util.Optional;

/**
 * @ClassName: KafkaController
 * @Description: TODO
 * @Author: MengMeng
 * @Date: 2020/12/23 12:59 下午
 * @Version: v1.0
 */
@RestController
public class KafkaController {

    @Autowired
    private KafkaTemplate kafkaTemplate;

    /**
     * @Description: providerKafka方法是 生产者
     * @param: []
     * @return: java.lang.String
     * @auther: MengMeng
     * @date: 2020/12/23 7:32 下午
     */
    @RequestMapping("/kafka")
    public String providerKafka() {
        ListenableFuture future = kafkaTemplate.send("meng-email","001", "xxx@163.com");
        future.addCallback(new ListenableFutureCallback<SendResult<Integer, String>>() {
            public void onFailure(Throwable throwable) {
                System.out.println("失败");
            }

            public void onSuccess(SendResult<Integer, String> integerStringSendResult) {
                System.out.println("成功");
            }
        });
//		JDK1.8写法
//		future.addCallback(o -> System.out.println("send-消息发送成功：" + message),
//				throwable -> System.out.println("消息发送失败：" + message));
        return "success";
    }

    /**
     * @Description: listen方法是 消费者
     * @param: [consumer]
     * @return: void
     * @auther: MengMeng
     * @date: 2020/12/23 7:37 下午
     */
    @KafkaListener(topics = {"meng-email"})
    public void listen(ConsumerRecord<?, ?> consumer) {
        Optional<?> kafkaMessage = Optional.ofNullable(consumer.value());
        if (kafkaMessage.isPresent()) {
            Object message = kafkaMessage.get();
            System.out.println("record =" + consumer);
            System.out.println("message =" + message);
            System.out.println("topic名称:" + consumer.topic() + ",key:" +
                    consumer.key() + "," + "分区位置:" + consumer.partition()
                    + ", 下标" + consumer.offset());
        }
    }
}

消费者：

yml：

spring:
  kafka:
    producer:
      bootstrap-servers: kafka-0:9092
      batch-size: 100
    consumer:
      group-id: meng_kafka_0
      auto-offset-reset: latest
      bootstrap-servers: kafka-0:9092
      auto-commit-interval: 1000
      enable-auto-commit: false
      max-poll-records: 1000
KafkaListener:
  feedback:
    topics: topic_1
    autoStartup: true
    # 消费者数量
    concurrency: 5
    # 单消费者线程数量
    parallelism: 10
    # 监听的 topic
    groupId: meng_kafka_0

config：

@Configuration
public class KafkaConsumerConfiguration {
    @Bean
    public KafkaListenerContainerFactory<?> batchFactory(ConsumerFactory consumerFactory) {
        ConcurrentKafkaListenerContainerFactory<Integer, String> factory =
                new ConcurrentKafkaListenerContainerFactory<>();
        factory.setConsumerFactory(consumerFactory);
        factory.getContainerProperties().setPollTimeout(180000);
        //设置为批量消费，每个批次数量在Kafka配置参数中设置
        factory.setBatchListener(true);
        factory.getContainerProperties().setAckMode(ContainerProperties.AckMode.MANUAL_IMMEDIATE);
        //设置手动提交ackMode
        return factory;
    }

    @Bean
    public ConsumerAwareListenerErrorHandler consumerAwareErrorHandler() {
        return (message, e, consumer) -> {
            e.printStackTrace();
            return null;
        };
    }

}

Listener:

/**
 * kafka 消费者
 *
 */
@Component
public class KafkaListener {
    private static final Gson GSON = new Gson();

    private static Integer concurrency;
    private static Boolean autoStartup;
    private static Integer parallelism;
    private static LinkedBlockingQueue<ForkJoinPool> forkJoinPoolList;

    final DisposeService disposeService;

    public KafkaFeedbackListener(DisposeService disposeService) {
        this.disposeService = disposeService;
    }

    @Value("${KafkaListener.feedback.concurrency}")
    public void setConcurrency(Integer concurrency) {
        KafkaFeedbackListener.concurrency = concurrency;
    }

    @Value("${KafkaListener.feedback.autoStartup}")
    public void setAutoStartup(Boolean autoStartup) {
        KafkaFeedbackListener.autoStartup = autoStartup;
    }

    @Value("${KafkaListener.feedback.parallelism}")
    public void setParallelism(Integer parallelism) {
        KafkaFeedbackListener.parallelism = parallelism;
    }

    /**
     * 初始化线程池
     */
    @PostConstruct
    private static void init() {
        log.debug("init forkJoinPoolList， concurrency is {}，parallelism is {},autoStartup is {}",
                concurrency, parallelism, autoStartup);
        if (null == forkJoinPoolList && autoStartup) {
            forkJoinPoolList = new LinkedBlockingQueue();
            for (int i = 0; i < concurrency; i++) {
                forkJoinPoolList.add(new ForkJoinPool(parallelism));
            }
        }
        log.debug("init forkJoinPoolList，end");
    }

    @KafkaListener(topics = {"#{'${KafkaListener.feedback.topics}'.split(',')}"},
            autoStartup = "#{'${KafkaListener.feedback.autoStartup}'}",
            groupId = "#{'${KafkaListener.feedback.groupId}'}",
            concurrency = "#{'${KafkaListener.feedback.concurrency}'}",
            containerFactory = "batchFactory",
            errorHandler = "consumerAwareErrorHandler")
    public void listenToKafka(List<ConsumerRecord<String, String>> records, Acknowledgment ack) throws ExecutionException, InterruptedException {
        long start = System.currentTimeMillis();

        ForkJoinPool finalForkJoinPool = getForkJoinPool();
        try {
            CompletableFuture[] completableFutures = records.stream().map(it -> CompletableFuture.runAsync(() -> {
                //demoService.demo(GSON.fromJson(it.value(), Demo.class));
            }, finalForkJoinPool).exceptionally(e -> {
                log.error("topic处理异常:{}", e);
                return null;
            })).toArray(CompletableFuture[]::new);

            CompletableFuture.allOf(completableFutures).exceptionally(e -> {
                log.error("topic allOf异常:{}", e);
                return null;
            }).get();
            ack.acknowledge();
        } finally {
            addForkJoinPool(finalForkJoinPool);
            long end = System.currentTimeMillis();
            log.info("feedback listener finally,time:{} ,avg:{}", end - start, (end - start) / records.size());
            records.clear();
        }
    }

    private ForkJoinPool getForkJoinPool() {
        return forkJoinPoolList.poll();
    }

    private void addForkJoinPool(ForkJoinPool forkJoinPool) {
        forkJoinPoolList.add(forkJoinPool);
    }

}

相关链接：

kafka是否自启动：springboot之kafkaListener手动开启与暂停_BrighteStar的博客-CSDN博客