Kafka常见使用实践

前言

在Java中引入并使用kafka，有很多方式，如可以直接使用原生kafka-clients这样的API；
也可以使用Spring对其的包装API，即spring-kafka

        <dependency>
            <groupId>org.springframework.kafka</groupId>
            <artifactId>spring-kafka</artifactId>
        </dependency>

KafkaTemplate是其生产者核心类，KafkaListener是其消费者核心注解；
或者使用更加抽象的SpringCloudStream等实现。

一、测试先行

本节首先学习一下如何使用spring-kafka-test进行kafka功能的测试，引入依赖

        <dependency>
            <groupId>org.springframework.kafka</groupId>
            <artifactId>spring-kafka-test</artifactId>
            <scope>test</scope>
        </dependency>

该依赖中提供了一个EmbeddedKafkaBroker模拟Kafka

消息监听与处理

@Component
public class MyKafkaListener {
   

    public BlockingQueue<String> kafkaMessages = new LinkedBlockingQueue<>();

    @KafkaListener(topics = {
   "apple-topic"})
    public void listen1(ConsumerRecord<?, ?> record) {
   
        Optional<?> kafkaMessage = Optional.ofNullable(record.value());
        if (kafkaMessage.isPresent()) {
   
            System.out.println("offset =" + record.offset());
            Instant instant = Instant.ofEpochMilli(record.timestamp());
            DateTimeFormatter df = DateTimeFormatter.ofPattern("yyyy-MM-dd hh:mm:ss");
            String time = LocalDateTime.ofInstant(instant, ZoneId.systemDefault()).format(df);
            System.out.println("time =" + time);
            String message = (String) kafkaMessage.get();
            System.out.println("message =" + message);
            kafkaMessages.add(message);
        }
    }
}

测试用例：

@RunWith(SpringRunner.class)
@SpringBootTest(properties =
        "spring.kafka.bootstrapServers:${" + EmbeddedKafkaBroker.SPRING_EMBEDDED_KAFKA_BROKERS + "}")
@EmbeddedKafka(topics = "apple-topic")
public class KafkaTest {
   
    private static final Logger log = LoggerFactory.getLogger(KafkaTest.class);

//    static {
   
//        System.setProperty(EmbeddedKafkaBroker.BROKER_LIST_PROPERTY, "spring.kafka.bootstrap-servers");
//    }

    @Autowired
    EmbeddedKafkaBroker broker;

    @Autowired
    private MyKafkaListener myKafkaListener;

    @Autowired
    KafkaTemplate<String, String> kafkaTemplate;

    @Test
    public void kafka() throws InterruptedException, ExecutionException {
   
        String message = "hello,world";
        ListenableFuture<SendResult<String, String>> future = kafkaTemplate
                .send("apple-topic", message);
        future.addCallback(new ListenableFutureCallback<SendResult<String, String>>() {
   
            @Override
            public void onFailure(Throwable throwable) {
   
                log.error("onFailure", throwable);
            }

            @Override
            public void onSuccess(SendResult<String, String> stringStringSendResult) {
   
                log.info("onSuccess {}", stringStringSendResult);
            }
        });
        log.info("result: {}", future.get());
        Assert.assertEquals(this.myKafkaListener.kafkaMessages.poll(10, TimeUnit.SECONDS