Springboot-kafka的集成以及使用方法

1.添加依赖包

<dependency>
    <groupId>org.springframework.kafka</groupId>
    <artifactId>spring-kafka</artifactId>
    <version>2.3.0.RELEASE</version>
</dependency>

2.yml配置文件配置

spring:
  application:
    name: kafka        #服务名称
  kafka:
    bootstrap-servers: IP:9092  # kafka服务器地址(可以多个,以"，"分割)
    consumer: # 指定一个默认的组名
      group-id: kafkaGroup
      # earliest:当各分区下有已提交的offset时，从提交的offset开始消费；无提交的offset时，从头开始消费
      # latest:当各分区下有已提交的offset时，从提交的offset开始消费；无提交的offset时，消费新产生的该分区下的数据
      # none:topic各分区都存在已提交的offset时，从offset后开始消费；只要有一个分区不存在已提交的offset，则抛出异常
      auto-offset-reset: earliest
      # key/value的反序列化
      key-deserializer: org.apache.kafka.common.serialization.StringDeserializer
      value-deserializer: org.apache.kafka.common.serialization.StringDeserializer
    producer:
      # key/value的序列化
      key-serializer: org.apache.kafka.common.serialization.StringSerializer
      value-serializer: org.apache.kafka.common.serialization.StringSerializer
      batch-size: 4096   # 批量抓取
      buffer-memory: 10240         # 缓存容量
      bootstrap-servers: IP:9092        # 服务器地址，可以多个逗号分隔

3.使用demo

import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.producer.Callback;
import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.clients.producer.RecordMetadata;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.kafka.annotation.EnableKafka;
import org.springframework.kafka.annotation.KafkaListener;
import org.springframework.kafka.annotation.PartitionOffset;
import org.springframework.kafka.annotation.TopicPartition;
import org.springframework.kafka.core.KafkaOperations;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.kafka.support.KafkaHeaders;
import org.springframework.messaging.handler.annotation.Header;
import org.springframework.messaging.handler.annotation.Payload;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RestController;

/**
 * @author: houkai
 * @Date: 2019/11/26 14:01
 */
@RestController
@EnableKafka
public class KafkaController {

    @Autowired
    private KafkaTemplate<String, String> kafkaTemplate;
    private Logger log = LoggerFactory.getLogger(KafkaController.class);

    /**消息发送*/
    @GetMapping("kafka")
    public String testKafka() {
        int iMax = 10;
        for (int i = 1; i < iMax; i++) {  //循环发6次kafka消息
            //topic名称  key   消息数据
            kafkaTemplate.send("test", "key_" + i, "data_" + i);
        }
        return "success";
    }

    /**
     * 订阅者
     * 对于同一个topic，不同组中的订阅者都会收到消息，即一个topic对应多个consumer,
     *      同一组中的订阅者只有一个consumer能够收到消息，即一个topic对应一个consumer
     */
    @KafkaListener(id = "customerID1", topics = "test", groupId = "group1")
    public void receive(ConsumerRecord<?, ?> consumer) {
        log.info("groupID: 1  topic名称:" + consumer.topic() + ",key:" + consumer.key() + ",分区位置:" + consumer.partition() + ", 下标" + consumer.offset() +"value:" + consumer.value());
    }
    /**订阅者*/
    @KafkaListener(id = "customerID2", topics = "test", groupId = "group2")
    public void receive2(ConsumerRecord<?, ?> consumer) {
        log.info("groupID:2  topic名称:" + consumer.topic() + ",key:" + consumer.key() + ",分区位置:" + consumer.partition() + ", 下标" + consumer.offset() +"value:" + consumer.value());
    }
    @KafkaListener(id = "customerID3", topics = "test", groupId = "group2")
    public void receive3(ConsumerRecord<?, ?> consumer) {
        log.info("groupID2  topic名称:" + consumer.topic() + ",key:" + consumer.key() + ",分区位置:" + consumer.partition() + ", 下标" + consumer.offset() +"value:" + consumer.value());
    }

    /**使用execute ProducerRecord 发送消息 */
    @GetMapping("execute")
    public void execute(){
        kafkaTemplate.execute(new KafkaOperations.ProducerCallback(){

            @Override
            public Object doInKafka(Producer producer) {
                ProducerRecord producerRecord = new ProducerRecord<String, String>("test", "test_data");
                //send方法需要回调的写法
                producer.send(producerRecord, new Callback() {
                    @Override
                    public void onCompletion(RecordMetadata recordMetadata, Exception e) {
                        //send success
                        if (null == e) {
                            log.info("send message success");
                            return;
                        }
                        log.error("send message failed");
                    }
                });
                return null;
            }
        });
    }
}