1.pom.xml
<dependency>
<groupId> org.apache.kafka</groupId>
<artifactId> kafka_2.10</artifactId>
<version> 0.8.0</version>
</dependency>
<dependency>
<groupId>org.springframework.kafka</groupId>
<artifactId>spring-kafka</artifactId>
<version>1.1.1.RELEASE</version>
</dependency>
<dependency>
<groupId>org.springframework.kafka</groupId>
<artifactId>spring-kafka-test</artifactId>
<version>1.0.2.RELEASE</version>
<scope>test</scope>
</dependency>
2.配置kafka(配置发送者,以及消费者,通过MyMessageListen)
package com.example.demo.n1;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.apache.kafka.common.serialization.StringSerializer;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.kafka.annotation.EnableKafka;
import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory;
import org.springframework.kafka.core.*;
import org.springframework.kafka.listener.KafkaMessageListenerContainer;
import org.springframework.kafka.listener.config.ContainerProperties;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Pattern;
@Configuration
@EnableKafka
public class KafkaConfig {
/* --------------producer configuration-----------------**/
@Bean
public Map<String, Object> producerConfigs() {
Map<String, Object> props = new HashMap<>();
props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "xxx:9092");
props.put(ProducerConfig.RETRIES_CONFIG, 0);
props.put(ProducerConfig.BATCH_SIZE_CONFIG, 16384);
props.put(ProducerConfig.LINGER_MS_CONFIG, 1);
props.put(ProducerConfig.BUFFER_MEMORY_CONFIG, 33554432);
props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
return props;
}
@Bean
public ProducerFactory<String, String> producerFactory() {
return new DefaultKafkaProducerFactory<>(producerConfigs());
}
/* --------------consumer configuration-----------------**/
@Bean
public Map<String, Object> consumerConfigs() {
Map<String, Object> props = new HashMap<>();
props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "xxx:9092");
props.put(ConsumerConfig.GROUP_ID_CONFIG, "0");
props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, true);
props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, 100);
props.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "15000");
props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
return props;
}
@Bean
ConcurrentKafkaListenerContainerFactory<String, String>
kafkaListenerContainerFactory() {
ConcurrentKafkaListenerContainerFactory<String, String> factory =
new ConcurrentKafkaListenerContainerFactory<>();
factory.setConsumerFactory(consumerFactory());
return factory;
}
@Bean
public ConsumerFactory<String, String> consumerFactory() {
return new DefaultKafkaConsumerFactory<>(consumerConfigs());
}
//实际执行消息消费的类,用于处理消息,做一些业务逻辑
@Bean
public MyMessageListener myMessageListener(){
return new MyMessageListener();
}
//消费者容器配置信息
@Bean
public ContainerProperties containerProperties(){
Pattern topicPattern = Pattern.compile("topic的名称"); //匹配满足正则的topic
ContainerProperties containerProperties = new ContainerProperties(topicPattern);//订阅满足正则表达式的topic
containerProperties.setMessageListener(myMessageListener());//订阅的topic的消息用myMessageListener去处理
return containerProperties;
}
@Bean
public KafkaMessageListenerContainer<String, String> kafkaMessageListenerContainer(){
return new KafkaMessageListenerContainer<>(consumerFactory(),containerProperties());
}
/* --------------kafka template configuration-----------------**/
@Bean
public KafkaTemplate<String,String> kafkaTemplate() {
KafkaTemplate<String, String> kafkaTemplate = new KafkaTemplate<>(producerFactory());
kafkaTemplate.setDefaultTopic("defaultTopic");
return kafkaTemplate;
}
}
3.配置消费者,里面可以写业务逻辑
package com.example.demo.n1;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.kafka.listener.MessageListener;
//监听器必须实现MessageListener这个接口中onMessage方法
public class MyMessageListener implements MessageListener<String, String> {
public final static Logger logger = LoggerFactory.getLogger(MyMessageListener.class);
@Override//此方法处理消息
public void onMessage(ConsumerRecord<String, String> data) {
String topic = data.topic();//消费的topic
logger.info("-------------recieve message from {} topic-------------", topic);
logger.info("partition:{}", String.valueOf(data.partition()));//消费的topic的分区
logger.info("offset:{}", String.valueOf(data.offset()));//消费者的位置
logger.info("get message from {} topic : {}", topic, data.value());//接收到的消息
}
}
4.测试控制器
package com.example.demo.n1;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.kafka.support.SendResult;
import org.springframework.util.concurrent.ListenableFuture;
import org.springframework.web.bind.annotation.*;
@RestController
public class TestController {
public final static Logger logger = LoggerFactory.getLogger(TestController.class);
@Autowired
private KafkaTemplate<String,String> kafkaTemplate;//kafkaTemplate相当于生产者
@RequestMapping(value = "/{topic}/send",method = RequestMethod.GET)
public void sendMeessage(
@RequestParam(value = "message",defaultValue = "hello world") String message,
@PathVariable final String topic) {
logger.info("start send message to {}",topic);
ListenableFuture<SendResult<String, String>> listenableFuture = kafkaTemplate.send(topic,message);//发送消息,topic不存在将自动创建新的topic
listenableFuture.addCallback(//添加成功发送消息的回调和失败的回调
result -> logger.info("send message to {} success",topic),
ex -> logger.info("send message to {} failure,error message:{}",topic,ex.getMessage()));
}
@RequestMapping(value = "/default/send",method = RequestMethod.GET)
public void sendMeessagedefault() {//发送消息到默认的topic
logger.info("start send message to default topic");
kafkaTemplate.sendDefault("你好,世界");
}
}
可以看到kafka消息发送是通过kafkaTemplate.send()