java集成kafka dome（包含批量消费）

我这里maven依赖用的版本比较低，大家可以自行选择不一定要用低版本的 

        <dependency>
            <groupId>org.springframework.kafka</groupId>
            <artifactId>spring-kafka</artifactId>
            <version>1.1.1.RELEASE</version>
        </dependency>

spring:
  kafka:
    bootstrap-servers: 外网ip:9092
     #producer:
     #retries: 0
     #batch-size: 16384
     #buffer-memory: 33554432
     #key-serializer: org.apache.kafka.common.serialization.StringSerializer
     #value-serializer: org.apache.kafka.common.serialization.StringSerializer
     #consumer:
     #group-id: test
     #auto-offset-reset: earliest
     #enable-auto-commit: true
     #auto-commit-interval: 100
     #key-deserializer: org.apache.kafka.common.serialization.StringDeserializer
     #value-deserializer: org.apache.kafka.common.serialization.StringDeserializer

生成者：

package com.xxx.admin.config;

import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.common.serialization.StringSerializer;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.kafka.core.DefaultKafkaProducerFactory;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.kafka.core.ProducerFactory;
import org.springframework.kafka.support.serializer.JsonSerializer;

import java.util.HashMap;
import java.util.Map;

@Configuration
public class KafkaProducerConfig {

    @Value(value = "${kafka.bootstrap-servers}")
    private String bootstrapAddress;


    @Bean
    public ProducerFactory<String, HegtBuyAccountOEunm> homeNewCanProducerFactory() {
        Map<String, Object> configProps = new HashMap<>();
        configProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapAddress);
        configProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
        configProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, JsonSerializer.class);
        return new DefaultKafkaProducerFactory<>(configProps);
    }

    @Bean
    public KafkaTemplate<String, HegtBuyAccountOEunm> homeNewCanKafkaTemplate() {
        return new KafkaTemplate<>(homeNewCanProducerFactory());
    }
}

生产者调用：

    @Autowired
    @Qualifier("homeNewCanKafkaTemplate")
    private KafkaTemplate kafkaTemplate;
    @Value("${kafka.topic.homeNewCan}")
    private String homeNewCanTopic;


    @Scheduled(cron = "*/30 * * * * ?")
    public void execute() throws ParseException {
           //这里代码有删减,参考当作例子就好
           HegtBuyAccountOEunm  hegtBuyAccountOEunm=new  HegtBuyAccountOEunm();
           kafkaTemplate.send(homeNewCanTopic, HegtBuyAccountOEunm.builder()
                            .buyAccount(hegtBuyAccountOEunm.getBuyAccount())
                            .uid(hegtBuyAccountOEunm.getUid())
                            .build());
    }

消费者：

package com.xxx.admin.config;

import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.kafka.annotation.EnableKafka;
import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory;
import org.springframework.kafka.core.ConsumerFactory;
import org.springframework.kafka.core.DefaultKafkaConsumerFactory;
import org.springframework.kafka.listener.config.ContainerProperties;
import org.springframework.kafka.support.serializer.JsonDeserializer;

import java.util.HashMap;
import java.util.Map;

@Configuration
@EnableKafka
public class KafkaConsumerConfig {
    @Value("${kafka.bootstrap-servers}")
    private String bootstrapServers;

  
    @Value("${kafka.group.homeNewCan}")
    private String HOMENEWCAN;

   

    @Bean
    public ConcurrentKafkaListenerContainerFactory<String, HegtBuyAccountOEunm> homeNewCanKafkaListenerContainerFactory() {
        Map<String, Object> configProps = new HashMap<>();
        configProps.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
        configProps.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
        configProps.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, JsonDeserializer.class);
        configProps.put(ConsumerConfig.GROUP_ID_CONFIG, HOMENEWCAN);
        configProps.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false);
        configProps.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, 100);//获取最大提交数量1
        ConcurrentKafkaListenerContainerFactory<String, HegtBuyAccountOEunm> factory = new ConcurrentKafkaListenerContainerFactory<>();
        ConsumerFactory<String, HegtBuyAccountOEunm> basicConsumerFactory = new DefaultKafkaConsumerFactory<>(configProps, new StringDeserializer(), new JsonDeserializer<>(HegtBuyAccountOEunm.class));
        factory.setConsumerFactory(basicConsumerFactory);
        factory.setBatchListener(true);
        factory.getContainerProperties().setPollTimeout(3000);//最大获取时间
        factory.getContainerProperties().setAckMode(AbstractMessageListenerContainer.AckMode.MANUAL_IMMEDIATE);//设置非自动提交
        //factory.setAutoStartup(false);// 禁止自动启动
        return factory;
    }
}

消费者监听：

    //初始化线程池
    private final static ExecutorService executor = Executors.newFixedThreadPool(50);

    /**
     * kafka(这里是用了多线程池批量消费处理)
     */
    @Transactional
    @KafkaListener(topics = "${kafka.topic.homeNewCan}", group = "${kafka.group.homeNewCan}", containerFactory = "homeNewCanKafkaListenerContainerFactory")
    public void gropKafaHomeNewCan(List<HegtBuyAccountOEunm> hegtBuyAccountOEunms,Acknowledgment ack) throws InterruptedException{
        ack.acknowledge();//提交offset
        hegtBuyAccountOEunms.forEach(item ->{
            executor.submit(() -> {
                try {
                    gropKafaHomeNewCanRealization(item);//这个是具体业务代码可自行替换
                } catch (Exception e) {
                    e.printStackTrace();
                }
            });
        });
        //executor.shutdown();
    }


    /**
     * kafka(这里单线程队列处理)
     */
    @Transactional
    @KafkaListener(topics = "${kafka.topic.homeNewCan}", group = "${kafka.group.homeNewCan}", containerFactory = "homeNewCanKafkaListenerContainerFactory")
    public void gropKafaHomeNewCan(HegtBuyAccountOEunm hegtBuyAccountOEunms) throws InterruptedException{
        LOGGER.info("处理kafka 参数：hegtBuyAccountOEunm："+hegtBuyAccountOEunms.toString());
    }