在JHipster(SpringBoot)中集成Kafka消息队列

在JHipster(SpringBoot)中集成Kafka消息队列

前置条件: Kafka服务器已经配置好。参见在Ubuntu机器上部署Kafka消息队列

使用Spring Cloud Stream 和Spring Kafka来进行集成。

步骤

  1. 添加依赖项
  2. 添加服务器和Topic配置
  3. 配置文件KafkaConfiguration.java
  4. 消息发布者服务 KafkaMessageSender.java
  5. 消息消费者KafkaConsumer.java

1 添加依赖

build.gradle中添加依赖项

compile "org.springframework.cloud:spring-cloud-stream"
compile "org.springframework.cloud:spring-cloud-stream-binder-kafka"
compile "org.springframework.kafka:spring-kafka:1.0.5.RELEASE"

2 添加服务器和Topic配置

application-dev.yml 或 ``中添加相应配置

spring:
    kafka.bootstrap-servers: 你的Kafka地址:9092
    #http://docs.spring.io/spring-cloud-stream/docs/Brooklyn.RELEASE/reference/html/_apache_kafka_binder.html
    cloud:  #spring.cloud.stream.kafka.binder.brokers
        stream:
            kafka:
                binder:
                    brokers: 你的Kafka地址
                    zk-nodes: 你的ZooKeeper地址
            bindings:
                output:
                    destination: topic-kafka-dev

3 配置文件KafkaConfiguration.java

package com.your.company.common.kafka;

import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.apache.kafka.common.serialization.StringSerializer;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.kafka.annotation.EnableKafka;
import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory;
import org.springframework.kafka.core.*;

import java.util.HashMap;
import java.util.Map;

/**
 * Kafka配置文件
 * https://www.codenotfound.com/2016/09/spring-kafka-consumer-producer-example.html
 */
@Configuration
@EnableKafka
public class KafkaConfiguration {

    @Value("${spring.kafka.bootstrap-servers}")
    private String bootstrapServers;

    //------------ Begin 生产者配置 ------------
    @Bean
    public Map<String, Object> producerConfigs() {
        Map<String, Object> props = new HashMap<>();
        // list of host:port pairs used for establishing the initial connections to the Kakfa cluster
        props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
        props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
        props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);

        return props;
    }

    @Bean
    public ProducerFactory<String, String> producerFactory() {
        return new DefaultKafkaProducerFactory<>(producerConfigs());
    }

    @Bean
    public KafkaTemplate<String, String> kafkaTemplate() {
        return new KafkaTemplate<>(producerFactory());
    }

    //------------ End 生产者配置 ------------

    //------------ Begin 消费者配置 ------------
    @Bean
    public Map<String, Object> consumerConfigs() {
        Map<String, Object> props = new HashMap<>();
        // list of host:port pairs used for establishing the initial connections to the Kakfa cluster
        props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
        props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
        props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
        props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false);
        props.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, 120000);
        props.put(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG, 180000);
        // allows a pool of processes to divide the work of consuming and processing records
        props.put(ConsumerConfig.GROUP_ID_CONFIG, "kafka_group");

        return props;
    }

    @Bean
    public ConsumerFactory<String, String> consumerFactory() {
        return new DefaultKafkaConsumerFactory<>(consumerConfigs());
    }

    @Bean
    public ConcurrentKafkaListenerContainerFactory<String, String> kafkaListenerContainerFactory() {
        ConcurrentKafkaListenerContainerFactory<String, String> factory =
            new ConcurrentKafkaListenerContainerFactory<>();
        factory.setConsumerFactory(consumerFactory());

        return factory;
    }
    //------------ End 消费者配置 ------------
}

4 消息发布者服务 KafkaMessageSender.java

简单起见,把消息转换成JSON字符串来发送和接收


package com.your.company.common.kafka;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.kafka.support.SendResult;
import org.springframework.stereotype.Service;
import org.springframework.util.concurrent.ListenableFuture;
import org.springframework.util.concurrent.ListenableFutureCallback;

/**
 * 消息发布者服务
 */
@Service
public class KafkaMessageSender {
    private static final Logger LOGGER = LoggerFactory.getLogger(KafkaMessageSender.class);

    @Autowired
    private KafkaTemplate<String, String> kafkaTemplate;

    public void send(String topic, String message) {
        // the KafkaTemplate provides asynchronous send methods returning a Future
        ListenableFuture<SendResult<String, String>> future = kafkaTemplate.send(topic, message);

        // register a callback with the listener to receive the result of the send asynchronously
        future.addCallback(new ListenableFutureCallback<SendResult<String, String>>() {

            @Override
            public void onSuccess(SendResult<String, String> result) {
                LOGGER.info("sent message='{}' with offset={}", message,
                    result.getRecordMetadata().offset());
            }

            @Override
            public void onFailure(Throwable ex) {
                LOGGER.error("unable to send message='{}'", message, ex);
            }
        });

        // or, to block the sending thread to await the result, invoke the future's get() method
    }

}

5 消息消费者KafkaConsumer.java

使用@KafkaListener(topics = "your_topic")注解来监听消息。

package com.your.company.common.kafka;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.kafka.annotation.KafkaListener;
import org.springframework.stereotype.Service;

/**
 * 消息消费者
 */
@Service
public class KafkaConsumer {
    private static final Logger LOGGER = LoggerFactory.getLogger(KafkaConsumer.class);

    //bin/kafka-console-consumer.sh --bootstrap-server 服务器IP:9092 --topic my_test --from-beginning
    @KafkaListener(topics = "my_test")
    public void receive(String message) {
        LOGGER.info("received message='{}'", message);
        System.out.println("receive message = " + message);
    }
}

在调试过程中,可以在服务器端,用命令行来接收消息或发布消息,分别调试发布和消费过程是否正常。

  • 0
    点赞
  • 0
    收藏
    觉得还不错? 一键收藏
  • 0
    评论
以下是使用Spring Boot实现Kafka生产者和死信队列的步骤: 1.在pom.xml文件添加Kafka和Spring Kafka依赖: ```xml <dependency> <groupId>org.springframework.kafka</groupId> <artifactId>spring-kafka</artifactId> <version>2.7.2</version> </dependency> <dependency> <groupId>org.apache.kafka</groupId> <artifactId>kafka-clients</artifactId> <version>2.8.0</version> </dependency> ``` 2.在application.properties文件配置Kafka: ```properties spring.kafka.bootstrap-servers=localhost:9092 spring.kafka.consumer.group-id=my-group spring.kafka.consumer.auto-offset-reset=earliest spring.kafka.consumer.properties.max.poll.interval.ms=60000 spring.kafka.consumer.properties.max.poll.records=100 spring.kafka.consumer.properties.max.partition.fetch.bytes=1048576 spring.kafka.consumer.properties.fetch.max.bytes=5242880 spring.kafka.consumer.properties.fetch.min.bytes=1 spring.kafka.consumer.properties.fetch.max.wait.ms=500 spring.kafka.consumer.properties.session.timeout.ms=30000 spring.kafka.consumer.properties.enable.auto.commit=true spring.kafka.consumer.properties.auto.commit.interval.ms=5000 spring.kafka.consumer.properties.isolation.level=read_committed spring.kafka.consumer.properties.value.deserializer=org.apache.kafka.common.serialization.StringDeserializer spring.kafka.producer.properties.value.serializer=org.apache.kafka.common.serialization.StringSerializer ``` 3.创建一个Kafka生产者: ```java @Service public class KafkaProducerService { @Autowired private KafkaTemplate<String, String> kafkaTemplate; public void sendMessage(String topic, String message) { kafkaTemplate.send(topic, message); } } ``` 4.创建一个Kafka消费者: ```java @Service public class KafkaConsumerService { @KafkaListener(topics = "my-topic") public void receiveMessage(String message) { // 处理消息 } @KafkaListener(id = "my-group", topics = "my-topic-dlq") public void receiveDlqMessage(String message) { // 处理死信队列消息 } } ``` 5.在application.properties文件配置死信队列: ```properties spring.kafka.listener.type=single spring.kafka.listener.poll-timeout=5000 spring.kafka.listener.concurrency=1 spring.kafka.listener.ack-mode=manual_immediate spring.kafka.listener.retry-template.max-attempts=3 spring.kafka.listener.retry-template.back-off=1000 spring.kafka.listener.retry-template.multiplier=2.0 spring.kafka.listener.retry-template.max-interval=60000 spring.kafka.listener.retry-template.exponential-backoff=true spring.kafka.listener.retry-template.initial-interval=1000 spring.kafka.listener.retry-template.retry-policy=SIMPLE_RETRY spring.kafka.listener.retry-template.stateful-retry=true spring.kafka.listener.retry-template.include-headers=true spring.kafka.listener.retry-template.max-time=60000 spring.kafka.listener.retry-template.recovery-callback-class=org.springframework.kafka.listener.SeekToCurrentErrorHandler spring.kafka.listener.retry-template.recovery-callback-method=recover spring.kafka.listener.retry-template.back-off-strategy=org.springframework.kafka.backoff.ExponentialBackOff spring.kafka.listener.retry-template.back-off-strategy.max-interval=60000 spring.kafka.listener.retry-template.back-off-strategy.initial-interval=1000 spring.kafka.listener.retry-template.back-off-strategy.multiplier=2.0 spring.kafka.listener.retry-template.back-off-strategy.max-attempts=3 spring.kafka.listener.retry-template.back-off-strategy.exponential=true spring.kafka.listener.retry-template.back-off-strategy.randomization-factor=0.5 spring.kafka.listener.retry-template.back-off-strategy.max-interval-factor=2.0 spring.kafka.listener.retry-template.back-off-strategy.min-interval=1000 spring.kafka.listener.retry-template.back-off-strategy.max-interval=60000 spring.kafka.listener.retry-template.back-off-strategy.delta=1000 spring.kafka.listener.retry-template.back-off-strategy.initial-duration=1000 spring.kafka.listener.retry-template.back-off-strategy.max-duration=60000 spring.kafka.listener.retry-template.back-off-strategy.multiplier=2.0 spring.kafka.listener.retry-template.back-off-strategy.randomization-factor=0.5 spring.kafka.listener.retry-template.back-off-strategy.max-interval-factor=2.0 spring.kafka.listener.retry-template.back-off-strategy.min-duration=1000 spring.kafka.listener.retry-template.back-off-strategy.max-duration=60000 spring.kafka.listener.retry-template.back-off-strategy.delta=1000 spring.kafka.listener.retry-template.back-off-strategy.initial-duration=1000 spring.kafka.listener.retry-template.back-off-strategy.max-duration=60000 spring.kafka.listener.retry-template.back-off-strategy.multiplier=2.0 spring.kafka.listener.retry-template.back-off-strategy.randomization-factor=0.5 spring.kafka.listener.retry-template.back-off-strategy.max-interval-factor=2.0 spring.kafka.listener.retry-template.back-off-strategy.min-duration=1000 spring.kafka.listener.retry-template.back-off-strategy.max-duration=60000 spring.kafka.listener.retry-template.back-off-strategy.delta=1000 spring.kafka.listener.retry-template.back-off-strategy.initial-duration=1000 spring.kafka.listener.retry-template.back-off-strategy.max-duration=60000 spring.kafka.listener.retry-template.back-off-strategy.multiplier=2.0 spring.kafka.listener.retry-template.back-off-strategy.randomization-factor=0.5 spring.kafka.listener.retry-template.back-off-strategy.max-interval-factor=2.0 spring.kafka.listener.retry-template.back-off-strategy.min-duration=1000 spring.kafka.listener.retry-template.back-off-strategy.max-duration=60000 spring.kafka.listener.retry-template.back-off-strategy.delta=1000 spring.kafka.listener.retry-template.back-off-strategy.initial-duration=1000 spring.kafka.listener.retry-template.back-off-strategy.max-duration=60000 spring.kafka.listener.retry-template.back-off-strategy.multiplier=2.0 spring.kafka.listener.retry-template.back-off-strategy.randomization-factor=0.5 spring.kafka.listener.retry-template.back-off-strategy.max-interval-factor=2.0 spring.kafka.listener.retry-template.back-off-strategy.min-duration=1000 spring.kafka.listener.retry-template.back-off-strategy.max-duration=60000 spring.kafka.listener.retry-template.back-off-strategy.delta=1000 spring.kafka.listener.retry-template.back-off-strategy.initial-duration=1000 spring.kafka.listener.retry-template.back-off-strategy.max-duration=60000 spring.kafka.listener.retry-template.back-off-strategy.multiplier=2.0 spring.kafka.listener.retry-template.back-off-strategy.randomization-factor=0.5 spring.kafka.listener.retry-template.back-off-strategy.max-interval-factor=2.0 spring.kafka.listener.retry-template.back-off-strategy.min-duration=1000 spring.kafka.listener.retry-template.back-off-strategy.max-duration=60000 spring.kafka.listener.retry-template.back-off-strategy.delta=1000 spring.kafka.listener.retry-template.back-off-strategy.initial-duration=1000 spring.kafka.listener.retry-template.back-off-strategy.max-duration=60000 spring.kafka.listener.retry-template.back-off-strategy.multiplier=2.0 spring.kafka.listener.retry-template.back-off-strategy.randomization-factor=0.5 spring.kafka.listener.retry-template.back-off-strategy.max-interval-factor=2.0 spring.kafka.listener.retry-template.back-off-strategy.min-duration=1000 spring.kafka.listener.retry-template.back-off-strategy.max-duration=60000 spring.kafka.listener.retry-template.back-off-strategy.delta=1000 spring.kafka.listener.retry-template.back-off-strategy.initial-duration=1000 spring.kafka.listener.retry-template.back-off-strategy.max-duration=60000 spring.kafka.listener.retry-template.back-off-strategy.multiplier=2.0 spring.kafka.listener.retry-template.back-off-strategy.randomization-factor=0.5 spring.kafka.listener.retry-template.back-off-strategy.max-interval-factor=2.0 spring.kafka.listener.retry-template.back-off-strategy.min-duration=1000 spring.kafka.listener.retry-template.back-off-strategy.max-duration=60000 spring.kafka.listener.retry-template.back-off-strategy.delta=1000 spring.kafka.listener.retry-template.back-off-strategy.initial-duration=1000 spring.kafka.listener.retry-template.back-off-strategy.max-duration=60000 spring.kafka.listener.retry-template.back-off-strategy.multiplier=2.0 spring.kafka.listener.retry-template.back-off-strategy.randomization-factor=0.5 spring.kafka.listener.retry-template.back-off-strategy.max-interval-factor=2.0 spring.kafka.listener.retry-template.back-off-strategy.min-duration=1000 spring.kafka.listener.retry-template.back-off-strategy.max-duration=60000 spring.kafka.listener.retry-template.back-off-strategy.delta=1000 spring.kafka.listener.retry-template.back-off-strategy.initial-duration=1000 spring.kafka.listener.retry-template.back-off-strategy.max-duration=60000 spring.kafka.listener.retry-template.back-off-strategy.multiplier=2.0 spring.kafka.listener.retry-template.back-off-strategy.randomization-factor=0.5 spring.kafka.listener.retry-template.back-off-strategy.max-interval-factor=2.0 spring.kafka.listener.retry-template.back-off-strategy.min-duration=1000 spring.kafka.listener.retry-template.back-off-strategy.max-duration=60000 spring.kafka.listener.retry-template.back-off-strategy.delta=1000 spring.kafka.listener.retry-template.back-off-strategy.initial-duration=1000 spring.kafka.listener.retry-template.back-off-strategy.max-duration=60000 spring.kafka.listener.retry-template.back-off-strategy.multiplier=2.0 spring.kafka.listener.retry-template.back-off-strategy.randomization-factor=0.5 spring.kafka.listener.retry-template.back-off-strategy.max-interval-factor=2.0 spring.kafka.listener.retry-template.back-off-strategy.min-duration=1000 spring.kafka.listener.retry-template.back-off-strategy.max-duration=60000 spring.kafka.listener.retry-template.back-off-strategy.delta=1000 spring.kafka.listener.retry-template.back-off-strategy.initial-duration=1000 spring.kafka.listener.retry-template.back-off-strategy.max-duration=60000 spring.kafka.listener.retry-template.back-off-strategy.multiplier=2.0 spring.kafka.listener.retry-template.back-off-strategy.randomization-factor=0.5 spring.kafka.listener.retry-template.back-off-strategy.max-interval-factor=2.0 spring.kafka.listener.retry-template.back-off-strategy.min-duration=1000 spring.kafka.listener.retry-template.back-off-strategy.max-duration=60000 spring.kafka.listener.retry-template.back-off-strategy.delta=1000 spring.kafka.listener.retry-template.back-off-strategy.initial-duration=1000 spring.kafka.listener.retry-template.back-off-strategy.max-duration=60000 spring.kafka.listener.retry-template.back-off-strategy.multiplier=2.0 spring.kafka.listener.retry-template.back-off-strategy.randomization-factor=0.5 spring.kafka.listener.retry-template.back-off-strategy.max-interval-factor=2.0 spring.kafka.listener.retry-template.back-off-strategy.min-duration=1000 spring.kafka.listener.retry-template.back-off-strategy.max-duration=60000 spring.kafka.listener.retry-template.back-off-strategy.delta=1000 spring.kafka.listener.retry-template.back-off-strategy.initial-duration=1000 spring.kafka.listener.retry-template.back-off-strategy.max-duration=60000 spring.kafka.listener.retry-template.back-off-strategy.multiplier=2.0 spring.kafka.listener.retry-template.back-off-strategy.randomization-factor=0.5 spring.kafka.listener.retry-template.back-off-strategy.max-interval-factor=2.0 spring.kafka.listener.retry-template.back-off-strategy.min-duration=1000 spring.kafka.listener.retry-template.back-off-strategy.max-duration=60000 spring.kafka.listener.retry-template.back-off-strategy.delta=1000 spring.kafka.listener.retry-template.back-off-strategy.initial-duration=1000 spring.kafka.listener.retry-template.back-off-strategy.max-duration=60000 spring.kafka.listener.retry-template.back-off-strategy.multiplier=2.0 spring.kafka.listener.retry-template.back-off-strategy.randomization-factor=0.5 spring.kafka.listener.retry-template.back-off-strategy.max-interval-factor=2.0 spring.kafka.listener.retry-template.back-off-strategy.min-duration=1000 spring.kafka.listener.retry-template.back-off-strategy.max-duration=60000 spring.kafka.listener.retry-template.back-off-strategy.delta=1000 spring.kafka.listener.retry-template.back-off-strategy.initial-duration=1000 spring.kafka.listener.retry-template.back-off-strategy.max-duration=60000 spring.kafka.listener.retry-template.back-off-strategy.multiplier=2.0 spring.kafka.listener.retry-template.back-off-strategy.randomization-factor=0.5 spring.kafka.listener.retry-template.back-off-strategy.max-interval-factor=2.0 spring.kafka.listener.retry-template.back-off-strategy.min-duration=1000 spring.kafka.listener.retry-template.back-off-strategy.max-duration=60000 spring.kafka.listener.retry-template.back-off-strategy.delta=1000 spring.kafka.listener.retry-template.back-off-strategy.initial-duration=1000 spring.kafka.listener.retry-template.back-off-strategy.max-duration=60000 spring.kafka.listener.retry-template.back-off-strategy.multiplier=2.0 spring.kafka.listener.retry-template.back-off-strategy.randomization-factor=0.5 spring.kafka.listener.retry-template.back-off-strategy.max-interval-factor=2.0 spring.kafka.listener.retry-template.back-off-strategy.min-duration=1000 spring.kafka.listener.retry-template.back-off-strategy.max-duration=60000 spring.kafka.listener.retry-template.back-off-strategy.delta=1000 spring.kafka.listener.retry-template.back-off-strategy.initial-duration=1000 spring.kafka.listener.retry-template.back-off-strategy.max-duration=60000 spring.kafka.listener.retry-template.back-off-strategy.multiplier=2.0 spring.kafka.listener.retry-template.back-off-strategy.randomization-factor=0.5 spring.kafka.listener.retry-template.back-off-strategy.max-interval-factor=2.0 spring.kafka.listener.retry-template.back-off-strategy.min-duration=1000 spring.kafka.listener.retry-template.back-off-strategy.max-duration=60000 spring.kafka.listener.retry-template.back-off-strategy.delta=1000 spring.kafka.listener.retry-template.back-off-strategy.initial-duration=1000 spring.kafka.listener.retry-template.back-off-strategy.max-duration=60000 spring.kafka.listener.retry-template.back-off-strategy.multiplier=2.0 spring.kafka.listener.retry-template.back-off-strategy.randomization-factor=0.5 spring.kafka.listener.retry-template.back-off-strategy.max-interval-factor=2.0 spring.kafka.listener.retry-template.back-off-strategy.min-duration=1000 spring.kafka.listener.retry-template.back-off-strategy.max-duration=60000 spring.kafka.listener.retry-template.back-off-strategy.delta

“相关推荐”对你有帮助么?

  • 非常没帮助
  • 没帮助
  • 一般
  • 有帮助
  • 非常有帮助
提交
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值