spring boot与kafka集成

配置文件application.properties

 

#kafka producer
spring.kafka.producer.bootstrap-servers=//多个节点用逗号分开
spring.kafka.producer.batch-size=65536
spring.kafka.producer.acks=1
spring.kafka.producer.key-serializer=org.apache.kafka.common.serialization.StringSerializer
spring.kafka.producer.value-serializer=org.apache.kafka.common.serialization.StringSerializer
spring.kafka.producer.retries=3
spring.kafka.producer.buffer-memory=4194304
# 指定默认消费者group id
#spring.kafka.consumer.group-id=
# 指定默认topic id
spring.kafka.template.default-topic=

pom.xml添加jar包依赖

<parent> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-parent</artifactId> <version>1.5.2.RELEASE</version> </parent>

<dependency>
    <groupId>org.springframework.boot</groupId>
    <artifactId>spring-boot-starter</artifactId>
    <version>1.5.2.RELEASE</version>
</dependency>

<dependency>
    <groupId>org.springframework.kafka</groupId>
    <artifactId>spring-kafka</artifactId>
    <version>1.2.0.RELEASE</version>
</dependency>

<dependency>
    <groupId>org.apache.kafka</groupId>
    <artifactId>kafka-clients</artifactId>
    <version>0.10.2.0</version>
</dependency>

启用kafka配置

 import org.springframework.context.annotation.Configuration; import org.springframework.kafka.annotation.EnableKafka; @Configuration @EnableKafka public class KafkaConsumerConfig { }

kafka生产者

import com.alibaba.fastjson.JSON;    import org.apache.log4j.Logger; import org.springframework.beans.factory.annotation.Value; import org.springframework.kafka.core.KafkaTemplate; import org.springframework.kafka.support.SendResult; import org.springframework.messaging.MessageHeaders; import org.springframework.stereotype.Component; import org.springframework.util.concurrent.ListenableFuture; import javax.annotation.Resource; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Random; import java.util.concurrent.ExecutionException; /**  * Copyright (c) 2017,All rights reserved.  *  * @author sky   * @version 1.0  * @Description:  * @date 2017-04-21 13:45  */ @Component("Producer") public class Producer { private static Logger log = Logger.getLogger(Producer.class); @Resource(name="kafkaTemplate") private KafkaTemplate kafkaTemplate; /**  * kafka发送消息模板  * @param topic 主题  * @param value messageValue  * @param ifPartition 是否使用分区 0是\1不是  * @param partitionNum 分区数 如果是否使用分区为0,分区数必须大于0  * @param role 角色:bbc app erp...  */  public Map<String,Object> sndMesForTemplate(String topic, Object value, String ifPartition, Integer partitionNum, String role){ String key = role+"-"+value.hashCode(); String valueString = JSON.toJSONString(value); if(ifPartition.equals("0")){ //表示使用分区  int partitionIndex = getPartitionIndex(key, partitionNum); ListenableFuture<SendResult<String, String>> result = kafkaTemplate.send(topic, partitionIndex, key, valueString); Map<String,Object> res = checkProRecord(result); return res; }else{ ListenableFuture<SendResult<String, String>> result = kafkaTemplate.send(topic, key, valueString); Map<String,Object> res = checkProRecord(result); return res; } } public Map<String,Object> sndMesForTemplate(MqMessage mqMessage){ MessageHeaders msgHeader = mqMessage.getHeaders(); String topic = kafkaTemplate.getDefaultTopic(); Map<String,Object> msg = new HashMap(); msg.put("application", EntityConstants.APPLICATION); msg.put("initTime",msgHeader.getTimestamp()); msg.put("body",mqMessage.getMessageBody()); mqMessage.setMessageBody(msg); Map<String,Object> map = MapUtil.createHashMap("id",msgHeader.getId(),"tags",mqMessage.getTags(),"topic",topic,"body",mqMessage.getMessageBody()); mqMessage.setMessageBody(map); log.info("Kafka -- 发送消息，"+ JSON.toJSONString(mqMessage)); ListenableFuture<SendResult<String, String>> result = kafkaTemplate.send(mqMessage); Map<String,Object> res = checkProRecord(result); log.info("Kafka -- 发送消息，SendResult :"+ JSON.toJSONString(res)); return res; } public int sndMesForTemplate(List<MqMessage> mqMessage){ if(mqMessage !=null){ for(MqMessage ms :mqMessage){ sndMesForTemplate(ms); } return mqMessage.size(); } return 0; } /**  * 根据key值获取分区索引  * @param key  * @param partitionNum  * @return  */  private int getPartitionIndex(String key, int partitionNum){ if (key == null) { Random random = new Random(); return random.nextInt(partitionNum); } else { int result = Math.abs(key.hashCode())%partitionNum; return result; } } /**  * 检查发送返回结果record  * @param res  * @return  */  @SuppressWarnings("rawtypes") private Map<String,Object> checkProRecord(ListenableFuture<SendResult<String, String>> res){ Map<String,Object> m = new HashMap<String,Object>(); if(res!=null){ try { SendResult r = res.get();//检查result结果集  /*检查recordMetadata的offset数据，不检查producerRecord*/  Long offsetIndex = r.getRecordMetadata().offset(); if(offsetIndex!=null && offsetIndex>=0){ m.put("code", EntityConstants.SUCCESS_CODE); m.put("message", EntityConstants.SUCCESS_MES); return m; }else{ m.put("code", EntityConstants.KAFKA_NO_OFFSET_CODE); m.put("message", EntityConstants.KAFKA_NO_OFFSET_MES); return m; } } catch (InterruptedException e) { e.printStackTrace(); m.put("code", EntityConstants.KAFKA_SEND_ERROR_CODE); m.put("message", EntityConstants.KAFKA_SEND_ERROR_MES); return m; } catch (ExecutionException e) { e.printStackTrace(); m.put("code", EntityConstants.KAFKA_SEND_ERROR_CODE); m.put("message", EntityConstants.KAFKA_SEND_ERROR_MES); return m; } }else{ m.put("code", EntityConstants.KAFKA_NO_RESULT_CODE); m.put("message", EntityConstants.KAFKA_NO_RESULT_MES); return m; } } } 

kafka消费者

 import org.apache.commons.logging.LogFactory; import org.springframework.kafka.annotation.KafkaListener; /**  * Copyright (c) 2017, All rights reserved.  *  * @author sky   * @version 1.0  * @Description:  * @date 2017-04-24 16:18  */ @Component public class KafkaConsumer { private static Log log = LogFactory.getLog(KafkaConsumer.class); @KafkaListener(topics = {"${consumer.topic}"}) public void kafkaConsumer(String data) { //具体的业务处理代码 } }