Kafka：集成SpringBoot

依赖

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
                             https://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>
    <parent>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-parent</artifactId>
        <version>2.2.8.RELEASE</version>
        <relativePath/> <!-- lookup parent from repository -->
    </parent>
    <groupId>com.lagou.kafka.demo</groupId>
    <artifactId>demo-02-springboot</artifactId>
    <version>0.0.1-SNAPSHOT</version>
    <name>demo-02-springboot</name>

    <description>Demo project for Spring Boot</description>
    <properties>
        <java.version>1.8</java.version>
    </properties>
    <dependencies>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
        </dependency>
        <dependency>
            <groupId>org.springframework.kafka</groupId>
            <artifactId>spring-kafka</artifactId>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-test</artifactId>
            <scope>test</scope>
            <exclusions>
                <exclusion>
                    <groupId>org.junit.vintage</groupId>
                    <artifactId>junit-vintage-engine</artifactId>
                </exclusion>
            </exclusions>
        </dependency>
        <dependency>
            <groupId>org.springframework.kafka</groupId>
            <artifactId>spring-kafka-test</artifactId>
            <scope>test</scope>
        </dependency>
    </dependencies>
    <build>
        <plugins>
            <plugin>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-maven-plugin</artifactId>
            </plugin>
        </plugins>
    </build>
</project>

 配置文件application.properties

spring.application.name=springboot-kafka
server.port=8080
# 用于建立初始连接的broker地址
spring.kafka.bootstrap-servers=192.168.25.129:9092
# producer用到的key和value的序列化类
spring.kafka.producer.key-serializer=org.apache.kafka.common.serialization.IntegerSerializer
spring.kafka.producer.value-serializer=org.apache.kafka.common.serialization.StringSerializer
# 默认的批处理记录数 16k
spring.kafka.producer.batch-size=16384
# 32MB的总发送缓存
spring.kafka.producer.buffer-memory=33554432
# consumer用到的key和value的反序列化类
spring.kafka.consumer.key-deserializer=org.apache.kafka.common.serialization.IntegerDeserializer
spring.kafka.consumer.value-deserializer=org.apache.kafka.common.serialization.StringDeserializer
# consumer的消费组id
spring.kafka.consumer.group-id=spring-kafka-consumer-group
# 是否自动提交消费者偏移量
spring.kafka.consumer.enable-auto-commit=true
# 每隔100ms向broker提交一次偏移量
spring.kafka.consumer.auto-commit-interval=100
# 如果该消费者的偏移量不存在，则自动设置为最早的偏移量
spring.kafka.consumer.auto-offset-reset=earliest

配置类KafkaConfig.java

import org.apache.kafka.clients.admin.NewTopic;

@Configuration
public class KafkaConfig {

    @Bean
    public NewTopic topic1() {
        // 创建主题
        // 参数一：设置主题名称 test
        // 参数二：分区数 5
        // 参数三：副本数 1
        return new NewTopic("test", 5, (short) 1);
    }

    // 配置全局admin
    @Bean
    public KafkaAdmin kafkaAdmin() {
        Map<String， Object> configs = new HashMap<>();
        configs.put("bootstrap.servers"，"192.168.25.129:9092");
        KafkaAdmin admin = new KafkaAdmin(configs);
        return admin;
    }

    @Bean
    @Autowired
    public KafkaTemplate<Integer, String> kafkaTemplate(ProducerFactory<Integer, String> producerFactory){

        // 覆盖ProducerFactory原有设置
        Map<String，Object> configsOverride = new HashMap<>();
        // 默认的批处理记录数 16k spring.kafka.producer.batch-size=16384 16k 现在设为 200
        configsOverrideput(ProducerConfig.BATCH SIZE CONFIG，200);
        KafkaTemplate<Integer, String> template = new KafkaTemplate<Integer, String>(
            producerFactory，
            configsOverride
        );
        return template
    }

}

生产者同步发送消息KafkaSyncProducerController.java

import org.apache.kafka.clients.producer.ProducerRecord;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.kafka.support.SendResult;
import org.springframework.util.concurrent.ListenableFuture;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import java.util.concurrent.ExecutionException;

@RestController
public class KafkaSyncProducerController {
    
    @Autowired
    private KafkaTemplate<Integer, String> template;
    
    @RequestMapping("send/sync/{message}")
    public String sendSync(@PathVariable String message) {
        
        ListenableFuture future = template.send(
            new ProducerRecord<Integer, String>(
                "test",	// 主题 test
                0,	    // 发送分区为 0
                1,	    // key为 1
                message)// value为 message
        );
        
        try {
            // 同步等待broker的响应
            Object o = future.get();
            SendResult<Integer, String> result = (SendResult<Integer, String>) o;
            System.out.println(result.getRecordMetadata().topic()
                               + result.getRecordMetadata().partition()
                               + result.getRecordMetadata().offset());
        } catch (InterruptedException e) {
            e.printStackTrace();
        } catch (ExecutionException e) {
            e.printStackTrace();
        }
        
        return "success";
    }
}

生产者异步发送消息KafkaAsyncProducerController

import org.apache.kafka.clients.producer.ProducerRecord;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.kafka.support.SendResult;
import org.springframework.util.concurrent.ListenableFuture;
import org.springframework.util.concurrent.ListenableFutureCallback;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;

@RestController
public class KafkaAsyncProducerController {
    
    @Autowired
    private KafkaTemplate<Integer, String> template;
    
    @RequestMapping("send/async/{message}")
    public String asyncSend(@PathVariable String message) {
        
        ProducerRecord<Integer, String> record = new ProducerRecord<Integer, String>(
            "test",	// 主题 test
            0,	    // 发送分区 0
            3,	    // key 3
            message	// value message
        );
        
        ListenableFuture<SendResult<Integer, String>> future = template.send(record);
        
        // 添加回调，异步等待响应
        future.addCallback(new ListenableFutureCallback<SendResult<Integer,String>>() {

            @Override
            public void onFailure(Throwable throwable) {
                System.out.println("发送失败: " + throwable.getMessage());
            }
            
            @Override
            public void onSuccess(SendResult<Integer, String> result) {
                System.out.println("发送成功：" +
                                   result.getRecordMetadata().topic() + "\t"
                                   + result.getRecordMetadata().partition() + "\t"
                                   + result.getRecordMetadata().offset());
            }
        });
        return "success";
    }
}

消费者MyConsumer.java

import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.springframework.kafka.annotation.KafkaListener;
import java.util.Optional;

@Component
public class MyConsumer {
    
    // 监听主题 test
    @KafkaListener(topics = "test")
    // onMessage 中的参数可以根据 KafkaListener注解 参数进行灵活添加
    // 比如ConsumerRecord 在KafkaListener注解 中存在 还有ack等
    public void onMessage(ConsumerRecord<Integer, String> record) {
        
        Optional<ConsumerRecord<Integer, String>> optional = Optional.ofNullable(record);
        
        if (optional.isPresent()) {
            System.out.println(
                record.topic() + "\t"
                + record.partition() + "\t"
                + record.offset() + "\t"
                + record.key() + "\t"
                + record.value());
        }
    }
}