基本配置:
spring.kafka.consumer.auto-offset-reset=earliest
spring.kafka.consumer.enable-auto-commit=false
spring.kafka.consumer.properties.fetch.max.wait.ms=60000
spring.kafka.consumer.properties.request.timeout.ms=70000
spring.kafka.consumer.properties.fetch.min.bytes=999999999
spring.kafka.listener.poll-timeout=60000
logging.level.org.springframework.kafka=debug
@Configuration
@EnableKafka
@EnableConfigurationProperties(KafkaProperties.class)
@Slf4j
public class KafkaConfiguration {
private final KafkaProperties properties;
public KafkaConfiguration(KafkaProperties properties) {
this.properties = properties;
}
// ==================== 消费者配置 ====================
private ConsumerFactory<Integer, String> canalConsumerFactory() {
return new DefaultKafkaConsumerFactory<>(consumerConfigs(canalKafkaServers));
}
private Map<String, Object> consumerConfigs(String servers) {
Map<String, Object> props = new HashMap<>(16);
props.putAll(properties.buildConsumerProperties());
props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, servers);
return props;
}
// ==================== 生产者配置 ====================
private ProducerFactory<Object, Object> producerFactory() {
return new DefaultKafkaProducerFactory<>(producerConfigs());
}
private Map<String, Object> producerConfigs() {
Map<String, Object> props = new HashMap<>(properties.buildProducerProperties());
props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "kafkaServers");
return props;
}
}
auto.offset.reset值详解:
earliest
当各分区下有已提交的offset时,从提交的offset开始消费;无提交的offset时,从头开始消费
latest
当各分区下有已提交的offset时,从提交的offset开始消费;无提交的offset时,消费新产生的该分区下的数据
none
topic各分区都存在已提交的offset时,从offset后开始消费;只要有一个分区不存在已提交的offset,则抛出异常