第四章 Spring和Kafka的整合笔记

最新推荐文章于 2024-04-15 22:57:52 发布

Hankin晚风

最新推荐文章于 2024-04-15 22:57:52 发布

阅读量679

点赞数

分类专栏： kafka进阶文章标签： Spring和Kafka的整合

本文链接：https://blog.csdn.net/m0_37661458/article/details/102809406

版权

本文详细介绍了Spring和SpringBoot与Kafka的整合过程，包括pom文件配置、统一配置、生产者与消费者端的代码实现。同时，通过kafka-traffic-shaping项目展示了如何利用Kafka进行削峰填谷的流量整形操作，提供了完整代码示例。

摘要由CSDN通过智能技术生成

一、与Spring集成

其实Spring和Kafka的集合，Spring是把Kafka当做一个消息中间件(MQ)来用，灵活性对比原生API还是没有那么灵活。具体代码实现，参见kafka-with-spring模块。

1、pom文件

<dependency>
    <groupId>org.springframework.kafka</groupId>
    <artifactId>spring-kafka</artifactId>
    <version>2.2.0.RELEASE</version>
</dependency>

2、统一配置

配置文件配置brokers地址：kafka.properties

#brokers集群

bootstrap.servers=localhost:9092

concurrency=3

3、生产者端

配置见代码applicationContext.xml中：

<context:property-placeholder location="classpath*:config/kafka.properties" />
<!-- 定义producer的参数 -->
<bean id="producerProperties" class="java.util.HashMap">
    <constructor-arg>
        <map>
            <entry key="bootstrap.servers" value="${bootstrap.servers}" />
            <entry key="key.serializer" value="org.apache.kafka.common.serialization.StringSerializer" />
            <entry key="value.serializer" value="org.apache.kafka.common.serialization.StringSerializer" />
        </map>
    </constructor-arg>
</bean>
<!-- 创建kafkatemplate需要使用的producerfactory bean -->
<bean id="producerFactory" class="org.springframework.kafka.core.DefaultKafkaProducerFactory">
    <constructor-arg>
        <ref bean="producerProperties"/>
    </constructor-arg>
</bean>
<!-- 发送监听器bean -->
<bean id="sendListener" class="cn.chj.service.SendListener" />

创建kafkatemplate bean，使用的时候只需要注入这个bean，即可使用template的send消息方法：

<bean id="kafkaTemplate" class="org.springframework.kafka.core.KafkaTemplate">
    <constructor-arg ref="producerFactory" />
    <constructor-arg name="autoFlush" value="true" />
    <!-- 配置发送监听器bean -->
    <property name="producerListener" ref="sendListener"></property>
</bean>
<!-- 1.定义consumer的参数 -->
<bean id="consumerProperties" class="java.util.HashMap">
    <constructor-arg>
        <map>
            <entry key="bootstrap.servers" value="${bootstrap.servers}" />
            <entry key="group.id" value="spring-kafka-group" />
            <entry key="key.deserializer" value="org.apache.kafka.common.serialization.StringDeserializer" />
            <entry key="value.deserializer" value="org.apache.kafka.common.serialization.StringDeserializer" />
        </map>
    </constructor-arg>
</bean>
<!-- 2.创建consumerFactory bean -->
<bean id="consumerFactory"
      class="org.springframework.kafka.core.DefaultKafkaConsumerFactory" >
    <constructor-arg>
        <ref bean="consumerProperties" />
    </constructor-arg>
</bean>
<!-- 3.定义消费实现类 -->
<bean id="kafkaConsumerService" class="cn.enjoyedu.service.KafkaConsumer" />
<!-- 4.消费者容器配置信息 -->
<bean id="containerProperties" class="org.springframework.kafka.listener.ContainerProperties">
    <constructor-arg name="topics">
        <list>
            <value>kafka-spring-topic</value>
        </list>
    </constructor-arg>
    <property name="messageListener" ref="kafkaConsumerService"></property>
</bean>
<!-- 5.消费者并发消息监听容器，执行doStart()方法 -->
<bean id="messageListenerContainer"
      class="org.springframework.kafka.listener.ConcurrentMessageListenerContainer" init-method="doStart" >
    <constructor-arg ref="consumerFactory" />
    <constructor-arg ref="containerProperties" />
    <property name="concurrency" value="${concurrency}" />
</bean>

KafkaController消息生产代码实现：

@Controller
@RequestMapping("/kafka")
public class KafkaController {
   @Autowired
   private KafkaTemplate<String,String> kafkaTemplate;
   /**
    * @param message
    */
   @ResponseBody
   @RequestMapping("spring")
   public String queueSender(@RequestParam("message")String message){
      String opt="";
      try {
         kafkaTemplate.send("kafka-spring-topic",message);
         opt = "suc";
      } catch (Exception e) {
         opt = e.getCause().toString();
      }
      return opt;
   }
   @ResponseBody
   @RequestMapping("springb")
   public String topicSender(@RequestParam("message")String message){
      String opt = "";
      try {
         kafkaTemplate.send("kafka-spring-topic-b",message);
         opt = "suc";
      } catch (Exception e) {
         opt = e.getCause().toString();
      }
      return opt;
   }
}

SendListener发送者确认：

public class SendListener implements ProducerListener {
    public void onSuccess(String topic, Integer partition, Object key, Object value, RecordMetadata recordMetadata) {
        System.out.println("offset:"+recordMetadata.offset()+"-"+"partition:"+recordMetadata.partition());
    }
    public void onError(String topic, Integer partition, Object key, Object value, Exception exception) {
    }
    public boolean isInterestedInSuccess() {
        return true;
    }
}

4、消费者端

见代码中applicationContext.xml

<!-- 消费者自行确认-1.定义consumer的参数 -->
<bean id="consumerPropertiesAck" class="java.util.HashMap">
    <constructor-arg>
        <map>
            <entry key="bootstrap.servers" value="${bootstrap.servers}" />
            <entry key="group.id" value="spring-kafka-group-ack" />
            <entry key="key.deserializer" value="org.apache.kafka.common.serialization.StringDeserializer" />
            <entry key="value.deserializer" value="org.apache.kafka.common.serialization.StringDeserializer" />
            <entry key="enable.auto.commit" value="false"/>
        </map>
    </constructor-arg>
</bean>
<!-- 消费者自行确认-2.创建consumerFactory bean -->
<bean id="consumerFactoryAck"  class="org.springframework.kafka.core.DefaultKafkaConsumerFactory" >
    <constructor-arg>
        <ref bean="consumerPropertiesAck" />
    </constructor-arg>
</bean>
<!-- 消费者自行确认-3.定义消费实现类 -->
<bean id="kafkaConsumerServiceAck" class="cn.chj.service.KafkaConsumerAck" />
<!-- 消费者自行确认-4.消费者容器配置信息 -->
<bean id="containerPropertiesAck" class="org.springframework.kafka.listener.ContainerProperties">
    <!-- topic -->
    <constructor-arg name="topics">
        <list>
            <value>kafka-spring-topic-b</value>
        </list>
    </constructor-arg>
    <property name="messageListener" ref="kafkaConsumerServiceAck" />
    <!-- 消费者自行确认模式 -->
    <property name="ackMode" value="MANUAL_IMMEDIATE"></property>
</bean>
<!-- 消费者自行确认-5.消费者并发消息监听容器，执行doStart()方法 -->
<bean id="messageListenerContainerAck" class="org.springframework.kafka.listener.ConcurrentMessageListenerContainer" init-method="doStart" >
    <constructor-arg ref="consumerFactoryAck" />
    <constructor-arg ref="containerPropertiesAck" />
    <property name="concurrency" value="${concurrency}" />
</bean>

KafkaConsumer消费者代码：

public class KafkaConsumer implements MessageListener<String,String> {
    public void onMessage(ConsumerRecord<String, String> data) {
        String name = Thread.currentThread().getName();
        System.out.println(name+"|"+String.format(
                "主题：%s，分区：%d，偏移量：%d，key：%s&#