Kafka的使用

pom.xml引入依赖

<dependency>
    <groupId>org.springframework.kafka</groupId>
    <artifactId>spring-kafka</artifactId>
</dependency>

简单使用

SimpleProducer.class

import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;

import java.util.Properties;

/**
 * FileName    : com.ronan.springbootthreaddemo.example.mq
 * Description :
 *
 * @author : ronan
 * @version : 1.0
 * Create Date : 2021/3/24 17:07
 **/
public class SimpleProducer {
    public static void main(String[] args) {
        Properties props = new Properties();
        props.put("bootstrap.servers","127.0.0.1:xxxx");

        props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
        props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");

        // 0 发出去确认 | 1 leader 落盘确认 | all(-1) 所有Follower同步完才确认
        props.put("acks","1");
        // 异常自动重试次数
        props.put("retries",3);
        // 多少条数据发送一次,默认16k
        props.put("batch.size",16384);
        // 批量发送的等待时间
        props.put("linger.ms",5);
        // 客户端缓冲区大小,默认32M,满了也会触发消息发送
        props.put("buffer.memory",33554432);
        // 获取元数据时生产者的阻塞时间,超时后抛出异常
        props.put("max.block.ms",3000);

        // 创建Sender线程
        KafkaProducer<String, String> producer = new KafkaProducer<>(props);

        for (int i = 0; i < 10; i++) {
            producer.send(new ProducerRecord<>("mytopic",Integer.toString(i),Integer.toString(i)));
        }

        producer.close();

    }
}

SimpleConsumer.class

import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;

import java.time.Duration;
import java.util.Arrays;
import java.util.Properties;

/**
 * FileName    : com.ronan.springbootthreaddemo.example.mq
 * Description :
 *
 * @author : ronan
 * @version : 1.0
 * Create Date : 2021/3/24 17:08
 **/
public class SimpleConsumer {
    public static void main(String[] args) {

        Properties props = new Properties();
        props.put("bootstrap.servers", "192.168.72.24:9092");
        props.put("group.id", "gp-test-group");
        // 是否自动提交偏移量,只有commit之后才更新消费组的offset
        props.put("enable.auto.commit", "true");
        // 消费者自动提交的间隔
        props.put("auto.commit.interval.ms", "1000");
        // 从最早的数据开始消费 earliest|latest|none
        props.put("auto.offset.reset", "earliest");
        props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");

        KafkaConsumer<String, String> consumer = new KafkaConsumer<>(props);
        // 订阅topic
        consumer.subscribe(Arrays.asList("mytopic"));


        try {
            while (true) {
                ConsumerRecords<String, String> records = consumer.poll(Duration.ofMillis(1000));
                for (ConsumerRecord<String, String> record : records) {
                    System.out.println("offset= " + record.offset() + ",key= " + record.key() + ",value= " + record.value() + ",partition=" + record.partition());
                }
            }
        } finally {
            consumer.close();
        }

    }
}

Spring整合Kafka使用

配置

# kafka的配置
# producer
spring.kafka.producer.retries=1
spring.kafka.producer.batch-size=16384
spring.kafka.producer.buffer-memory=33554432
spring.kafka.producer.acks=all
spring.kafka.producer.properties.linger.ms=5
spring.kafka.producer.key-serializer=org.apache.kafka.common.serialization.StringSerializer
spring.kafka.producer.value-serializer=org.apache.kafka.common.serialization.StringSerializer

# consumer
spring.kafka.consumer.auto-offset-reset=earliest
spring.kafka.consumer.enable-auto-commit=true
spring.kafka.consumer.auto-commit-interval=1000
spring.kafka.consumer.key-deserializer=org.apache.kafka.common.serialization.StringDeserializer
spring.kafka.consumer.value-deserializer=org.apache.kafka.common.serialization.StringDeserializer

KafkaProducer.class

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.stereotype.Component;
import org.springframework.web.bind.annotation.RequestParam;

/**
 * FileName    : com.ronan.springbootthreaddemo.provider
 * Description :
 *
 * @author : ronan
 * @version : 1.0
 * Create Date : 2021/3/24 17:47
 **/
@Component
public class KafkaProducer {
    
    @Autowired
    private KafkaTemplate<String,Object> kafkaTemplate;
    
    public String send(@RequestParam String msg) {
        kafkaTemplate.send("springboottopic",msg);
        return "ok";
    }
}

ConsumerListener.class

import org.springframework.kafka.annotation.KafkaListener;
import org.springframework.stereotype.Component;

/**
 * FileName    : com.ronan.springbootthreaddemo.listener
 * Description : 消费者
 *
 * @author : ronan
 * @version : 1.0
 * Create Date : 2021/3/24 17:44
 **/
@Component
public class ConsumerListener {

    @KafkaListener(topics = {"springboottopic"},groupId = "springboottopic-group")
    public void onMessage(String msg) {
        System.out.println("-----收到消息:" + msg);
    }
}