kafka --- 消费者

主要关注以下几个点：1、offset的提交  2、消息的幂等性

maven配置

        <dependency>
            <groupId>org.springframework.kafka</groupId>
            <artifactId>spring-kafka</artifactId>
        </dependency>

 application.yml配置：

  kafka:
    bootstrap-servers: localhost:9092
    consumer:
      #key-deserializer: org.apache.kafka.common.serialization.StringDeserializer
      #value-deserializer: org.apache.kafka.common.serialization.StringDeserializer
      # earliest latest
      auto-offset-reset: earliest
      #关闭自动提交 改由spring-kafka提交
      #enable-auto-commit: false
      #auto-commit-interval: 100
      #批量消费 一次接收的最大数量
      max-poll-records: 500
      # 自定义的消费topic
      topic:
        yc_test_consumer_topic: testTopic
      # 默认消费组id
      group-pre: yc-

 java config:

@Configuration
public class KafkaConsumerConfig {

    @Value("${spring.kafka.bootstrap-servers}")
    private String kafkaServer;
    @Value("${spring.kafka.consumer.group-pre}")
    private String groupPre;
    @Value("${spring.kafka.consumer.auto-offset-reset}")
    private String offsetPosition;
    @Value("${spring.kafka.consumer.max-poll-records}")
    private Integer maxPollRecords;

    @Value("${spring.kafka.consumer.topic.yc_test_consumer_topic}")
    private String topicWaybillPickup;

    @Bean
    KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<Integer, String>> waybillPickupContainerFactory() {
        ConcurrentKafkaListenerContainerFactory<Integer, String>
                factory = new ConcurrentKafkaListenerContainerFactory<>();
        // 设置消费者工厂
        factory.setConsumerFactory(waybillPickupConsumerFactory());
        // 消费者组中线程数量
        factory.setConcurrency(2);
        commonFactoryAssignment(factory);
        return factory;
    }

    @Bean
    public ConsumerFactory<Integer, String> waybillPickupConsumerFactory() {
        return new DefaultKafkaConsumerFactory<>(waybillPickupConsumerConfigs());
    }

    /**
     * 消费工厂公共配置
     * @param factory
     */
    private void commonFactoryAssignment(ConcurrentKafkaListenerContainerFactory<Integer, String> factory) {
        // 拉取超时时间
        factory.getContainerProperties().setPollTimeout(3000);
        // 批次监听
        factory.setBatchListener(true);
        // 批次提交
        factory.getContainerProperties().setAckMode((ContainerProperties.AckMode.MANUAL));
    }

    @Bean
    public Map<String, Object> waybillPickupConsumerConfigs() {
        Map<String, Object> propsMap = new HashMap<>();
        commonConfigAssignment(propsMap);
        propsMap.put(ConsumerConfig.GROUP_ID_CONFIG, groupPre + topicWaybillPickup);
        return propsMap;
    }

    /**
     * 消费者config公共配置
     * @param propsMap
     */
    private void commonConfigAssignment(Map<String, Object> propsMap) {
        propsMap.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaServer);
        // 是否自动提交offset偏移量(默认true)
        propsMap.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false);
        propsMap.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, maxPollRecords);
        // 键的反序列化方式
        propsMap.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
        // 值的反序列化方式
        propsMap.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
        // offset偏移量规则设置：
        // (1)、earliest：当各分区下有已提交的offset时，从提交的offset开始消费；无提交的offset时，从头开始消费
        // (2)、latest：当各分区下有已提交的offset时，从提交的offset开始消费；无提交的offset时，消费新产生的该分区下的数据
        // (3)、none：topic各分区都存在已提交的offset时，从offset后开始消费；只要有一个分区不存在已提交的offset，则抛出异常
        propsMap.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, offsetPosition);
    }

}

offset提交

看上配置里面有个  enable.auto.commit 参数，如果 enable.auto.commit = false 则手动提交
如果 enable.auto.commit = true自动提交（默认配置）
在手动提交的情况下，可以设置提交的方式
factory.getContainerProperties().setAckMode((ContainerProperties.AckMode.MANUAL));
这种方式需要我们在代码中执行   ack.acknowledge();来对offset进行提交，这是最常用的方式，还有几种方式，大家可以去了解以下

@Component
@Slf4j
public class ConsumerService {

    @KafkaListener(
            topics = "${spring.kafka.consumer.topic.yc_test_consumer_topic}",
            containerFactory = "waybillPickupContainerFactory"
    )
    public void omsWaybillPickupListen(List<ConsumerRecord<String, String>> records, Acknowledgment ack) {
        try {

            for (ConsumerRecord<String, String> record : records) {
                if (log.isDebugEnabled()) {
                    log.debug("Listen Method: erpDispatchSendTrunkLineListen, Thread ID: {}, Topic: {}, Partition: {}, Data: {}", Thread.currentThread().getId(), record.topic(), record.partition(), record.value());
                }
                JSONObject jsonObject = null;
                try {
                    jsonObject = JSON.parseObject(record.value());
                } catch (Exception e) {
                    log.error("该条记录不是binlog:" + record.value(), e);
                    continue;
                }
                log.info("==========================="+JSONObject.toJSONString(jsonObject));
            }
            ack.acknowledge();
        } catch (Exception e) {
            log.error("数据处理异常", e);
        }
    }
}

消息的幂等性

可以看消息重复的解决方式