Kafka配置

 

基本配置：

spring.kafka.consumer.auto-offset-reset=earliest
spring.kafka.consumer.enable-auto-commit=false
spring.kafka.consumer.properties.fetch.max.wait.ms=60000
spring.kafka.consumer.properties.request.timeout.ms=70000
spring.kafka.consumer.properties.fetch.min.bytes=999999999
spring.kafka.listener.poll-timeout=60000

logging.level.org.springframework.kafka=debug

@Configuration
@EnableKafka
@EnableConfigurationProperties(KafkaProperties.class)
@Slf4j
public class KafkaConfiguration {

    private final KafkaProperties properties;

    public KafkaConfiguration(KafkaProperties properties) {
        this.properties = properties;
    }


    // ====================  消费者配置  ====================

    private ConsumerFactory<Integer, String> canalConsumerFactory() {
        return new DefaultKafkaConsumerFactory<>(consumerConfigs(canalKafkaServers));
    }

    private Map<String, Object> consumerConfigs(String servers) {
        Map<String, Object> props = new HashMap<>(16);
        props.putAll(properties.buildConsumerProperties());
        props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, servers);
        return props;
    }

    // ====================  生产者配置  ====================

    private ProducerFactory<Object, Object> producerFactory() {
        return new DefaultKafkaProducerFactory<>(producerConfigs());
    }

    private Map<String, Object> producerConfigs() {
        Map<String, Object> props = new HashMap<>(properties.buildProducerProperties());
       
        props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "kafkaServers");
        return props;
    }
}

 

auto.offset.reset值详解：

参考 Kafka auto.offset.reset值详解

earliest
当各分区下有已提交的offset时，从提交的offset开始消费；无提交的offset时，从头开始消费
latest
当各分区下有已提交的offset时，从提交的offset开始消费；无提交的offset时，消费新产生的该分区下的数据
none
topic各分区都存在已提交的offset时，从offset后开始消费；只要有一个分区不存在已提交的offset，则抛出异常