PS: 会持续更新springboot系列,希望大家多多支持. >_<
kafka版本间配置会存在一些差异,需依据实际版本进行适配.具体参数描述此处不做讲解.
此例为: kafka-clients-0.10.1.1
maven:
<dependency>
<groupId>org.springframework.kafka</groupId>
<artifactId>spring-kafka</artifactId>
<version>1.0.6.RELEASE</version> <!-- 如果kafka版本为0.10,不需要添加, 如果为0.9,需要加上版本 -->
</dependency>
properties:
kafka.consumer.servers=集群地址,集群地址
kafka.consumer.enable.auto.commit=false
kafka.consumer.session.timeout=6000
kafka.consumer.auto.commit.interval=100
kafka.consumer.auto.offset.reset=earliest
kafka.consumer.topic=topic
kafka.consumer.group.id=groupid
kafka.consumer.concurrency=5
kafka.producer.servers=
kafka.producer.retries=0
kafka.producer.batch.size=4096
kafka.producer.linger=1
kafka.producer.buffer.memory=40960
kafka_消费配置:
KafkaConsumerConfig.java:
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.kafka.annotation.EnableKafka;
import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory;
import org.springframework.kafka.config.KafkaListenerContainerFactory;
import org.springframework.kafka.core.ConsumerFactory;
import org.springframework.kafka.core.DefaultKafkaConsumerFactory;
import org.springframework.kafka.listener.ConcurrentMessageListenerContainer;
import com.ky.rdpcs.comm.kafka.KafkaRec;
import java.util.HashMap;
import java.util.Map;
@Configuration
@EnableKafka
public class KafkaConsumerConfig {
@Value("${kafka.consumer.servers}")
private String servers;
@Value("${kafka.consumer.enable.auto.commit}")
private boolean enableAutoCommit;
@Value("${kafka.consumer.session.timeout}")
private String sessionTimeout;
@Value("${kafka.consumer.auto.commit.interval}")
private String autoCommitInterval;
@Value("${kafka.consumer.group.id}")
private String groupId;
@Value("${kafka.consumer.auto.offset.reset}")
private String autoOffsetReset;
@Value("${kafka.consumer.concurrency}")
private int concurrency;
@Bean
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
factory.setConsumerFactory(consumerFactory());
factory.setConcurrency(concurrency);
factory.getContainerProperties().setPollTimeout(1500);
return factory;
}
public ConsumerFactory<String, String> consumerFactory() {
return new DefaultKafkaConsumerFactory<>(consumerConfigs());
}
public Map<String, Object> consumerConfigs() {
Map<String, Object> propsMap = new HashMap<>();
propsMap.put("key.deserializer.encoding", "gb18030");
propsMap.put("value.deserializer.encoding", "gb18030");
propsMap.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, servers);
propsMap.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, enableAutoCommit);
propsMap.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, autoCommitInterval);
propsMap.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, sessionTimeout);
propsMap.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, autoOffsetReset);
propsMap.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
propsMap.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
propsMap.put(ConsumerConfig.GROUP_ID_CONFIG, groupId);
propsMap.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, autoOffsetReset);
propsMap.put(ConsumerConfig.RECEIVE_BUFFER_CONFIG, "33554432");
propsMap.put(ConsumerConfig.FETCH_MIN_BYTES_CONFIG, "33554432");
propsMap.put(ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG, "33554432");
propsMap.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, "1000");
return propsMap;
}
@Bean
public KafkaRec listener() {
return new KafkaRec();
}
}
Rec.java
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.kafka.annotation.KafkaListener;
import org.springframework.stereotype.Component;
/**
* 接收kafka消息入口
*/
@Component
public class KafkaRec {
/**
* 监听器自动执行该方法 消费消息 自动提交offset 执行业务代码 (high level api 不提供offset管理,不能指定offset进行消费)
* topic: 一类消息,如page view,click行为等。
* partition: topic的物理分组,每个partition都是一个有序队列。
* offset: 一个连续的用于定位被追加到分区的每一个消息的序列号,最大值为64位的long大小,19位数字字符长度。
* value: 消息体
*/
@KafkaListener(topics = {"topic"})
public void onMessage(ConsumerRecord<String, String> record) {
String topic = record.topic();
String value = record.value();
long offset = record.offset();
int partition = record.partition();
long startTime = System.currentTimeMillis();
}
}
由于时间原因,此次先只写消费端,生产端后续更新.