【kafka】手动拉取消息

最新推荐文章于 2024-05-09 00:46:31 发布

ooooooooooooooxiaosu

最新推荐文章于 2024-05-09 00:46:31 发布

阅读量951

点赞数

分类专栏： RabbitMQ&ActiveMQ&Kafka 文章标签： kafka

本文链接：https://blog.csdn.net/xiaoxiao_su123/article/details/126991370

版权

RabbitMQ&ActiveMQ&Kafka 专栏收录该内容

9 篇文章 0 订阅

订阅专栏

需求描述

消费失败的消息，要重新拉取进行消费处理。

解决方法

import com.xxx.capitalplat.common.enums.ReturnCode;
import com.xxx.capitalplat.common.exception.KafkaException;
import com.xxx.capitalplat.modules.app.service.KafkaService;
import lombok.extern.slf4j.Slf4j;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;

import java.time.Duration;
import java.util.Collections;
import java.util.Iterator;
import java.util.Properties;

@Slf4j
@Service
public class KafkaServiceImpl implements KafkaService {

    private static final long DURATION_TIMEOUT = 5L;
    private static final int COUNT = 1;

    @Value("${spring.kafka.bootstrap-servers}")
    private String bootstrapServers;
    @Value("${spring.kafka.consumer.group-id}")
    private String groupId;
    @Value("${spring.kafka.consumer.auto-commit-interval}")
    private String autoCommitInterval;
    @Value("${spring.kafka.consumer.auto-offset-reset}")
    private String autoOffsetReset;


    @Override
    public ConsumerRecord<String, String> pull(String topic, int partition, int offset) {
        log.info("拉取kafka消息，topic：{}，partition：{}，offset：{}", topic, partition, offset);
        Properties props = new Properties();
        props.put("bootstrap.servers", bootstrapServers);
        props.put("group.id", groupId);
        props.put("enable.auto.commit", "true");
        props.put("auto.commit.interval.ms", autoCommitInterval);
        props.put("session.timeout.ms", "30000");
        props.put("auto.offset.reset", autoOffsetReset);
        props.put("key.deserializer", StringDeserializer.class.getName());
        props.put("value.deserializer", StringDeserializer.class.getName());
        log.info("拉取kafka消息配置：{}", props);

        try (KafkaConsumer<String, String> consumer = new KafkaConsumer<>(props)) {
            TopicPartition topicPartition = new TopicPartition(topic, partition);
            consumer.assign(Collections.singletonList(topicPartition));
            consumer.seek(topicPartition, offset);
            ConsumerRecords<String, String> consumerRecords = consumer.poll(Duration.ofSeconds(DURATION_TIMEOUT));

            if (consumerRecords.count() == COUNT) {
                Iterator<ConsumerRecord<String, String>> iterator = consumerRecords.iterator();
                ConsumerRecord<String, String> consumerRecord = iterator.next();
                log.info("拉取kafka消息结果，topic：{}，partition：{}，offset:{}，msg：{}", consumerRecord.topic(), consumerRecord.partition(), consumerRecord.offset(), consumerRecord.value());
                return consumerRecord;
            } else {
                throw new KafkaException(ReturnCode.DATA_STATUS_ERROR, "拉取kafka消息数量错误");
            }
        } catch (Exception e) {
            throw new KafkaException(ReturnCode.FAIL, "拉取kafka消息错误");
        }
    }


}