一、maven
<spring.version>4.3.13.RELEASE</spring.version>
<dependency>
<groupId>org.springframework.kafka</groupId>
<artifactId>spring-kafka</artifactId>
<version>1.3.8.RELEASE</version>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>2.0.1</version>
</dependency>
二、kafka consumer 配置
import org.apache.kafka.clients.producer.ProducerConfig;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.kafka.annotation.EnableKafka;
import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory;
import org.springframework.kafka.config.KafkaListenerContainerFactory;
import org.springframework.kafka.core.ConsumerFactory;
import org.springframework.kafka.core.DefaultKafkaConsumerFactory;
import org.springframework.kafka.listener.ConcurrentMessageListenerContainer;
import java.util.HashMap;
import java.util.Map;
/**
* spring 注解配置 - kafka consumer 配置
*/
@Configuration
@EnableKafka
public class SpringKafkaConsumerConfig {
@Value("${kafka.bootstrap.servers}")
private String bootstrapServer;
@Value("${kafka.consumer.group}")
private String consumerGroup;
@Value("${kafka.consumer.concurrency:3}")
private int consumerConcurrency;
@Value("${kafka.consumer.fetch.max.bytes:1048576}")
private int fetchMaxBytes;
@Value("${kafka.consumer.max.poll.records:10000}")
private int maxPollRecords;
@Value("${kafka.client.id}")
private String clientId;
@Value("${kafka.sasl.jaas.config}")
private String saslJaasConfig;
@Bean
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<Integer, String>> kafkaListenerContainerFactory() {
ConcurrentKafkaListenerContainerFactory<Integer, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
factory.setConsumerFactory(consumerFactory());
factory.setConcurrency(consumerConcurrency);
factory.getContainerProperties().setPollTimeout(30000);
return factory;
}
@Bean
public ConsumerFactory<Integer, String> consumerFactory() {
return new DefaultKafkaConsumerFactory<>(consumerConfigs());
}
@Bean
public Map<String, Object> consumerConfigs() {
Map<String, Object> props = new HashMap<>();
props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServer);
props.put("fetch.max.bytes",fetchMaxBytes);
props.put("max.poll.records",maxPollRecords);
props.put("enable.auto.commit", true);
props.put("group.id", consumerGroup);
props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
props.put(ProducerConfig.CLIENT_ID_CONFIG, clientId);
props.put("security.protocol", "SASL_PLAINTEXT");
props.put("sasl.mechanism", "SCRAM-SHA-256");
props.put("sasl.jaas.config", saslJaasConfig);
return props;
}
}
三、接收kafka消息
import lombok.extern.slf4j.Slf4j;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.kafka.annotation.KafkaListener;
import org.springframework.stereotype.Component;
/**
* 接收kafka消息
*/
@Slf4j
@Component
public class KafkaListener {
@Autowired
private MessageHandler messageHandler;
/**
*
*
* @param record
*/
@KafkaListener(topics = "${kafka.topic}")
public void onMessageDataSync(ConsumerRecord<String, String> record) {
String value = record.value();
log.info("接收到record: {}", record);
try {
messageHandler.handleData(value);
log.info("process success, record: {}", record);
} catch (Exception ex) {
log.error("process error record: {}", record, ex);
}
}
}