在JHipster(SpringBoot)中集成Kafka消息队列

在JHipster(SpringBoot)中集成Kafka消息队列

前置条件: Kafka服务器已经配置好。参见在Ubuntu机器上部署Kafka消息队列

使用Spring Cloud Stream 和Spring Kafka来进行集成。

步骤

1. 添加依赖项
2. 添加服务器和Topic配置
3. 配置文件KafkaConfiguration.java
4. 消息发布者服务 KafkaMessageSender.java
5. 消息消费者KafkaConsumer.java

1 添加依赖

在build.gradle中添加依赖项

compile "org.springframework.cloud:spring-cloud-stream"
compile "org.springframework.cloud:spring-cloud-stream-binder-kafka"
compile "org.springframework.kafka:spring-kafka:1.0.5.RELEASE"

2 添加服务器和Topic配置

在application-dev.yml 或 ``中添加相应配置

spring:
    kafka.bootstrap-servers: 你的Kafka地址:9092
    #http://docs.spring.io/spring-cloud-stream/docs/Brooklyn.RELEASE/reference/html/_apache_kafka_binder.html
    cloud:  #spring.cloud.stream.kafka.binder.brokers
        stream:
            kafka:
                binder:
                    brokers: 你的Kafka地址
                    zk-nodes: 你的ZooKeeper地址
            bindings:
                output:
                    destination: topic-kafka-dev

3 配置文件KafkaConfiguration.java

package com.your.company.common.kafka;

import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.apache.kafka.common.serialization.StringSerializer;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.kafka.annotation.EnableKafka;
import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory;
import org.springframework.kafka.core.*;

import java.util.HashMap;
import java.util.Map;

/**
 * Kafka配置文件
 * https://www.codenotfound.com/2016/09/spring-kafka-consumer-producer-example.html
 */
@Configuration
@EnableKafka
public class KafkaConfiguration {

    @Value("${spring.kafka.bootstrap-servers}")
    private String bootstrapServers;

    //------------ Begin 生产者配置 ------------
    @Bean
    public Map<String, Object> producerConfigs() {
        Map<String, Object> props = new HashMap<>();
        // list of host:port pairs used for establishing the initial connections to the Kakfa cluster
        props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
        props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
        props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);

        return props;
    }

    @Bean
    public ProducerFactory<String, String> producerFactory() {
        return new DefaultKafkaProducerFactory<>(producerConfigs());
    }

    @Bean
    public KafkaTemplate<String, String> kafkaTemplate() {
        return new KafkaTemplate<>(producerFactory());
    }

    //------------ End 生产者配置 ------------

    //------------ Begin 消费者配置 ------------
    @Bean
    public Map<String, Object> consumerConfigs() {
        Map<String, Object> props = new HashMap<>();
        // list of host:port pairs used for establishing the initial connections to the Kakfa cluster
        props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
        props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
        props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
        props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false);
        props.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, 120000);
        props.put(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG, 180000);
        // allows a pool of processes to divide the work of consuming and processing records
        props.put(ConsumerConfig.GROUP_ID_CONFIG, "kafka_group");

        return props;
    }

    @Bean
    public ConsumerFactory<String, String> consumerFactory() {
        return new DefaultKafkaConsumerFactory<>(consumerConfigs());
    }

    @Bean
    public ConcurrentKafkaListenerContainerFactory<String, String> kafkaListenerContainerFactory() {
        ConcurrentKafkaListenerContainerFactory<String, String> factory =
            new ConcurrentKafkaListenerContainerFactory<>();
        factory.setConsumerFactory(consumerFactory());

        return factory;
    }
    //------------ End 消费者配置 ------------
}

4 消息发布者服务 KafkaMessageSender.java

简单起见，把消息转换成JSON字符串来发送和接收

package com.your.company.common.kafka;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.kafka.support.SendResult;
import org.springframework.stereotype.Service;
import org.springframework.util.concurrent.ListenableFuture;
import org.springframework.util.concurrent.ListenableFutureCallback;

/**
 * 消息发布者服务
 */
@Service
public class KafkaMessageSender {
    private static final Logger LOGGER = LoggerFactory.getLogger(KafkaMessageSender.class);

    @Autowired
    private KafkaTemplate<String, String> kafkaTemplate;

    public void send(String topic, String message) {
        // the KafkaTemplate provides asynchronous send methods returning a Future
        ListenableFuture<SendResult<String, String>> future = kafkaTemplate.send(topic, message);

        // register a callback with the listener to receive the result of the send asynchronously
        future.addCallback(new ListenableFutureCallback<SendResult<String, String>>() {

            @Override
            public void onSuccess(SendResult<String, String> result) {
                LOGGER.info("sent message='{}' with offset={}", message,
                    result.getRecordMetadata().offset());
            }

            @Override
            public void onFailure(Throwable ex) {
                LOGGER.error("unable to send message='{}'", message, ex);
            }
        });

        // or, to block the sending thread to await the result, invoke the future's get() method
    }

}

5 消息消费者KafkaConsumer.java

使用@KafkaListener(topics = "your_topic")注解来监听消息。

package com.your.company.common.kafka;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.kafka.annotation.KafkaListener;
import org.springframework.stereotype.Service;

/**
 * 消息消费者
 */
@Service
public class KafkaConsumer {
    private static final Logger LOGGER = LoggerFactory.getLogger(KafkaConsumer.class);

    //bin/kafka-console-consumer.sh --bootstrap-server 服务器IP:9092 --topic my_test --from-beginning
    @KafkaListener(topics = "my_test")
    public void receive(String message) {
        LOGGER.info("received message='{}'", message);
        System.out.println("receive message = " + message);
    }
}

在调试过程中，可以在服务器端，用命令行来接收消息或发布消息，分别调试发布和消费过程是否正常。