使用spring集成的kafka收发消息

原创 2016年08月29日 11:10:05

1. 引入maven依赖

<dependency>
	<groupId>org.springframework.integration</groupId>
	<artifactId>spring-integration-kafka</artifactId>
	<version>${spring-integration-kafka.version}</version>
</dependency>

2. 生产者的xml配置

<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
       xmlns:int="http://www.springframework.org/schema/integration"
       xmlns:int-kafka="http://www.springframework.org/schema/integration/kafka"
       xsi:schemaLocation="http://www.springframework.org/schema/integration/kafka http://www.springframework.org/schema/integration/kafka/spring-integration-kafka.xsd
		http://www.springframework.org/schema/integration http://www.springframework.org/schema/integration/spring-integration.xsd
		http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd">

    <int:channel id="outWriteBackLemmaRecordChannel" />

    <int-kafka:outbound-channel-adapter id="kafkaOutboundChannelAdapter"
                                        kafka-template="kafkaTemplate"
                                        auto-startup="true"
                                        channel="outWriteBackLemmaRecordChannel"
                                        order="3"
                                        topic="writeBackLemmaRecordTopic">
        <int-kafka:request-handler-advice-chain>
            <bean class="org.springframework.integration.handler.advice.RequestHandlerCircuitBreakerAdvice" />
        </int-kafka:request-handler-advice-chain>
    </int-kafka:outbound-channel-adapter>

    <bean id="kafkaTemplate" class="org.springframework.kafka.core.KafkaTemplate">
        <constructor-arg>
            <bean class="org.springframework.kafka.core.DefaultKafkaProducerFactory">
                <constructor-arg>
                    <map>
                        <entry key="bootstrap.servers" value="1.1.1.1:9092,2.2.2.2:9092"/>
                        <entry key="retries" value="10"/>
                        <entry key="batch.size" value="16384"/>
                        <entry key="linger.ms" value="1"/>
                        <entry key="buffer.memory" value="33554432"/>
                        <entry key="key.serializer" value="org.apache.kafka.common.serialization.IntegerSerializer"/>
                        <entry key="value.serializer" value="org.apache.kafka.common.serialization.StringSerializer"/>
                    </map>
                </constructor-arg>
            </bean>
        </constructor-arg>
        <constructor-arg name="autoFlush" value="true"/>
        <property name="defaultTopic" value="writeBackLemmaRecordTopic"/>
    </bean>

    <bean id="kafkaProducerService"
          class="com.soso.baike.admin.service.kafka.producer.impl.KafkaProducerServiceImpl"/>
</beans>

针对DefaultKafkaProducerFactory 的参数,本公司其实是配置注册到了zookeeper上,针对开发环境,预发环境,线上环境的配置是不同的,所以zookeeper上分别针对不同的环境注册了三套配置文件,发布的时候,会根据要发布的环境去zookeeper上拉取对应环境的配置文件,从而填充DefaultKafkaProducerFactory的构造参数


3. 发送消息

发送消息是上述配置文件中配置的KafkaProducerServiceImpl类

package com.soso.baike.admin.service.kafka.producer.impl;

import com.soso.baike.admin.service.kafka.producer.IKafkaProducerService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.kafka.core.KafkaTemplate;


public class KafkaProducerServiceImpl implements IKafkaProducerService {
    private Logger logger = LoggerFactory.getLogger("kafka");

    @Autowired
    private KafkaTemplate<Integer, String> kafkaTemplate;<span style="white-space:pre">	</span>//这个已经在上述xml文件中配置

    @Override
    public void sendMessage(String topic, String data) {
        logger.info("the message is to be send by kafka is : topic = {}, data = {}", topic, data);
        kafkaTemplate.setDefaultTopic(topic);
        kafkaTemplate.sendDefault(data);
    }

    @Override
    public void sendMessage(String topic, int key, String data) {
        logger.info("the message is to be send by kafka is : topic = {}, data = {}", topic, data);
        kafkaTemplate.setDefaultTopic(topic);
        kafkaTemplate.sendDefault(key, data);
    }
}

4.  消费者xml配置

<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
       xmlns:int-kafka="http://www.springframework.org/schema/integration/kafka"
       xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
		http://www.springframework.org/schema/integration/kafka http://www.springframework.org/schema/integration/kafka/spring-integration-kafka.xsd">

    <int-kafka:message-driven-channel-adapter
            id="kafkaMessageDrivenChannelAdapter"
            listener-container="kafkaMessageListenerContainer"
            auto-startup="true"
            phase="100"
            send-timeout="5000"
            channel="nullChannel"
            message-converter="messagingMessageConverter"
            error-channel="errorChannel"/>

    <bean id="messagingMessageConverter" class="org.springframework.kafka.support.converter.MessagingMessageConverter"/>

    <bean id="kafkaMessageListenerContainer" class="org.springframework.kafka.listener.KafkaMessageListenerContainer">
        <constructor-arg>
            <bean class="org.springframework.kafka.core.DefaultKafkaConsumerFactory">
                <constructor-arg>
                    <map>
                        <entry key="bootstrap.servers" value="${kafka.consumer.bootstrap.servers}"/>
                        <entry key="group.id" value="${kafka.consumer.group.id}"/>
                        <entry key="enable.auto.commit" value="${kafka.consumer.enable.auto.commit}"/>
                        <entry key="auto.commit.interval.ms" value="${kafka.consumer.auto.commit.interval.ms}"/>
                        <entry key="session.timeout.ms" value="${kafka.consumer.session.timeout.ms}"/>
                        <entry key="key.deserializer" value="${kafka.consumer.key.deserializer}"/>
                        <entry key="value.deserializer" value="${kafka.consumer.value.deserializer}"/>
                    </map>
                </constructor-arg>
            </bean>
        </constructor-arg>
        <constructor-arg>
            <bean class="org.springframework.kafka.listener.config.ContainerProperties">
                <constructor-arg name="topics" value="writeBackLemmaRecordTopic"/>
            </bean>
        </constructor-arg>
    </bean>

    <!-- 实际执行消息消费的类 -->
    <bean id="kafkaConsumerService"
          class="com.soso.baike.admin.service.kafka.consumer.impl.KafkaConsumerServiceImpl"/>
</beans>

上述DefaultKafkaConsumerFactory的构造参数就是在配置文件中配置的,这里你可以直接替换成实际的参数而不用配置文件


5. 接收消息类是上述配置文件中配置的KafkaConsumerServiceImpl类,代码如下:

package com.soso.baike.admin.service.kafka.consumer.impl;

import com.soso.baike.admin.constant.KafkaConstants;
import com.soso.baike.admin.lmaimp.DummyUser;
import com.soso.baike.admin.service.kafka.consumer.IKafkaConsumerService;
import com.soso.baike.audit.Auditors;
import com.soso.baike.audit.db.LemmaAuditDao;
import com.soso.baike.audit.lemma.LemmaRecord;
import com.soso.baike.audit.lemma.LemmaWriteBackOp;
import com.soso.baike.domain.IdConvert;
import org.apache.commons.lang.math.NumberUtils;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.kafka.listener.KafkaMessageListenerContainer;
import org.springframework.kafka.listener.config.ContainerProperties;

import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

/**
 * Created by zhangyongguang on 2016/6/30.
 */
public class KafkaConsumerServiceImpl implements IKafkaConsumerService, InitializingBean {
    private Logger logger = LoggerFactory.getLogger("kafka");

    @Autowired
    private KafkaMessageListenerContainer kafkaMessageListenerContainer;
    @Autowired
    private LemmaWriteBackOp lemmaWriteBackOp;

    private int threadNum = 8;
    private int maxQueueSize = 2000;
    private ExecutorService executorService = new ThreadPoolExecutor(threadNum,
            threadNum, 0L, TimeUnit.MILLISECONDS,
            new ArrayBlockingQueue<Runnable>(maxQueueSize),
            new ThreadPoolExecutor.CallerRunsPolicy());

    @Override
    public void onMessage(ConsumerRecord<Integer, String> record) {
        logger.info("===============processMessage===============");
        logger.info("the kafka message is arriving with topic = {}, partition = {}, key = {}, value = {}",
                new Object[]{record.topic(), record.partition(), record.key(), record.value()});
<span style="white-space:pre">	</span>//这里收到消息后,开启了一个线程来处理<span style="white-space:pre">	</span>
        executorService.execute(new Runnable() {
            @Override
            public void run() {
                String msg = record.value();
                
            }
        });
    }

    @Override<span style="white-space:pre">	</span>//设置监听
    public void afterPropertiesSet() throws Exception {
        ContainerProperties containerProperties = kafkaMessageListenerContainer.getContainerProperties();

        if (null != containerProperties) {
            containerProperties.setMessageListener(this);
        }
    }
}


版权声明:本文为博主原创文章,未经博主允许不得转载。

相关文章推荐

Kafka _0.10.1.0 liunx安装

1. 下载、解压Kafka安装包 wget http://archive.apache.org/dist/kafka/0.10.1.0/kafka_2.10-0.10.1.0.tgz tar -xz...

文件编码转换程序

import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.File; import java.io...

Kafka——使用spring进行集成

生产者: <beans xmlns="http://www.springframework.org/schema/beans" xmlns:xsi="http://www.w3.org/200...

kafka_0.10.1.0与Spring4的集成

准备工作 kafka版本:kafka_2.10-0.10.1.0 spring版本:spring4.3 pom文件配置 ...

spring集成kafka实现producer和consumer

spring集成kafka实现producer和consumer

kafka 工具类

package com.zsb.test.util; import java.util.ArrayList; import java.util.HashMap; import java.util.L...
内容举报
返回顶部
收藏助手
不良信息举报
您举报文章:深度学习:神经网络中的前向传播和反向传播算法推导
举报原因:
原因补充:

(最多只允许输入30个字)