1、pom.xml 依赖
<dependency>
<groupId>org.springframework.kafka</groupId>
<artifactId>spring-kafka</artifactId>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
</dependency>
2、application.yml 和 application.properties
【注意】:若是kafka不需要配置加密账号密码,则
kafka.inner.security-protocol、kafka.inner.sasl-mechanism、kafka.producer.sasl-jaas-config、kafka.consumer.sasl-jaas-config
均不需要配置,这点在下述额config中会判断
#kafka
kafka.inner.bootstrap-servers=xxx.xx.xx.1:29092, xxx.xx.xx.2:29092, xxx.xx.xx.3:29092
kafka.inner.security-protocol=SASL_PLAINTEXT
kafka.inner.sasl-mechanism=PLAIN
#=============== producer =======================
# 写入失败时,重试次数。当leader节点失效,一个repli节点会替代成为leader节点,此时可能出现写入失败,
# 当retris为0时,produce不会重复。retirs重发,此时repli节点完全成为leader节点,不会产生消息丢失。
kafka.producer.retries=0
# 生产者jaas配置账号密码
kafka.producer.sasl-jaas-config=org.apache.kafka.common.security.plain.PlainLoginModule required username=\"producer\" password=\"123456\";
# 每次批量发送消息的数量,produce积累到一定数据,一次发送
kafka.producer.batch-size=16384
# produce积累数据一次发送,缓存大小达到buffer.memory就发送数据
kafka.producer.buffer-memory=33554432
kafka.producer.linger-ms=5
#=============== consumer =======================
# 指定默认消费者group id --> 由于在kafka中,同一组中的consumer不会读取到同一个消息,依靠groud.id设置组名
kafka.consumer.group-id=kafkaGroup
# 消费者jaas配置账号密码
kafka.consumer.sasl-jaas-config=org.apache.kafka.common.security.plain.PlainLoginModule required username=\"consumer\" password=\"123456\";
# smallest和largest才有效,如果smallest重新0开始读取,如果是largest从logfile的offset读取。一般情况下我们都是设置smallest
kafka.consumer.auto-offset-reset=earliest
# enable.auto.commit:true --> 设置自动提交offset
kafka.consumer.enable-auto-commit=true
#如果'enable.auto.commit'为true,则消费者偏移自动提交给Kafka的频率(以毫秒为单位),默认值为5000。
kafka.consumer.auto-commit-interval=100
kafka.consumer.max-poll-records=1000
3、KafkaConfiguration.java
@Value("${kafka.inner.bootstrap-servers}")
private String bootstrapServers;
@Value("${kafka.inner.security-protocol}")
private String kafkaSecurityProtocol;
@Value("${kafka.inner.sasl-mechanism}")
private String kafkaSASLMechanism;
@Value("${kafka.consumer.group-id}")
private String groupId;
@Value("${kafka.consumer.sasl-jaas-config}")
private String kafkaConsumerSASLJaasConfig;
@Value("${kafka.consumer.auto-offset-reset}")
private String autoOffsetReset;
@Value("${kafka.consumer.max-poll-records}")
private String maxPollRecords;
@Value("${kafka.producer.retries}")
private String producerRetries;
@Value("${kafka.producer.sasl-jaas-config}")
private String kafkaProducerSASLJaasConfig;
@Value("${kafka.producer.batch-size}")
private String producerBatchSize;
@Value("${kafka.producer.linger-ms}")
private String producerLingerMs;
@Value("${kafka.producer.buffer-memory}")
private String bufferMemory;
@Bean
public KafkaListenerContainerFactory<?> batchFactory(){
ConcurrentKafkaListenerContainerFactory<String, String> factory = new
ConcurrentKafkaListenerContainerFactory<>();
factory.setConsumerFactory(new DefaultKafkaConsumerFactory<>(consumerConfigs()));
factory.setBatchListener(true); // 开启批量监听
return factory;
}
@Bean
public Map<String, Object> consumerConfigs() {
Map<String, Object> config = new HashMap<>();
//kafka地址
config.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
//组id
config.put(ConsumerConfig.GROUP_ID_CONFIG, groupId);
config.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, autoOffsetReset);
config.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, maxPollRecords);
config.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "100");
config.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "15000");
config.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
config.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
if (!StringUtils.isEmpty(kafkaSecurityProtocol) && !StringUtils.isEmpty(kafkaSASLMechanism)
&& !StringUtils.isEmpty(kafkaConsumerSASLJaasConfig)) {
config.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, kafkaSecurityProtocol);
config.put(SaslConfigs.SASL_MECHANISM, kafkaSASLMechanism);
config.put("sasl.jaas.config", kafkaConsumerSASLJaasConfig);
}
return config;
}
@Bean
public ProducerFactory<String, String> producerFactory() {
Map<String, Object> properties = new HashMap<>();
properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
properties.put(ProducerConfig.RETRIES_CONFIG, producerRetries); // 重试,0为不启用重试机制
properties.put(ProducerConfig.BATCH_SIZE_CONFIG, producerBatchSize); // 控制批处理大小,单位为字节,默认为16384
properties.put(ProducerConfig.LINGER_MS_CONFIG, producerLingerMs); // 批量发送,延迟为5毫秒,启用该功能能有效减少生产者发送消息次数,从而提高并发量,默认为0
properties.put(ProducerConfig.BUFFER_MEMORY_CONFIG, bufferMemory); // 生产者可以使用的总内存字节来缓冲等待发送到服务器的记录,默认为33554432,使用默认值即可
properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
if (!StringUtils.isEmpty(kafkaSecurityProtocol) && !StringUtils.isEmpty(kafkaSASLMechanism)
&& !StringUtils.isEmpty(kafkaProducerSASLJaasConfig)) {
properties.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, kafkaSecurityProtocol);
properties.put(SaslConfigs.SASL_MECHANISM, kafkaSASLMechanism);
properties.put("sasl.jaas.config", kafkaProducerSASLJaasConfig);
}
return new DefaultKafkaProducerFactory<>(properties);
}
@Bean
public KafkaTemplate<String, String> kafkaTemplate() {
return new KafkaTemplate<>(producerFactory());
}
4、kafka生产者
# 注入在config配置好的kafka对象
@Autowired
private KafkaTemplate<String, String> kafkaTemplate;
# 直接使用send方法发送
kafkaTemplate.send(topic,message);
kafkaTemplate.send(topic,key,message);
kafkaTemplate.sendDefault(message);
kafkaTemplate.sendDefault(key,message);
5、kafka消费者
# 单条接收数据
@KafkaListener(topics = "topic")
public void onMessage(Sring message){
logger.info("接受数据:{}", message);
}
# 批量接收数据,这里需要配置containerFactory,而我们在上述的kafkaConfig文件中配置了该消费者
@KafkaListener(topics = "topic",containerFactory = "batchFactory")
public void onMessage(List<ConsumerRecord<?, ?>> records){
List<Map<String,String>> mapList = new ArrayList<>();
for (ConsumerRecord<?, ?> record : records) {
Optional<?> kafkaMessage = Optional.ofNullable(record.value());
logger.info("Received: " + record);
if (kafkaMessage.isPresent()) {
Object message = record.value();
logger.info("接受数据:{}", message);
}
}
}