需求描述
消费失败的消息,要重新拉取进行消费处理。
解决方法
import com.xxx.capitalplat.common.enums.ReturnCode;
import com.xxx.capitalplat.common.exception.KafkaException;
import com.xxx.capitalplat.modules.app.service.KafkaService;
import lombok.extern.slf4j.Slf4j;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import java.time.Duration;
import java.util.Collections;
import java.util.Iterator;
import java.util.Properties;
@Slf4j
@Service
public class KafkaServiceImpl implements KafkaService {
private static final long DURATION_TIMEOUT = 5L;
private static final int COUNT = 1;
@Value("${spring.kafka.bootstrap-servers}")
private String bootstrapServers;
@Value("${spring.kafka.consumer.group-id}")
private String groupId;
@Value("${spring.kafka.consumer.auto-commit-interval}")
private String autoCommitInterval;
@Value("${spring.kafka.consumer.auto-offset-reset}")
private String autoOffsetReset;
@Override
public ConsumerRecord<String, String> pull(String topic, int partition, int offset) {
log.info("拉取kafka消息,topic:{},partition:{},offset:{}", topic, partition, offset);
Properties props = new Properties();
props.put("bootstrap.servers", bootstrapServers);
props.put("group.id", groupId);
props.put("enable.auto.commit", "true");
props.put("auto.commit.interval.ms", autoCommitInterval);
props.put("session.timeout.ms", "30000");
props.put("auto.offset.reset", autoOffsetReset);
props.put("key.deserializer", StringDeserializer.class.getName());
props.put("value.deserializer", StringDeserializer.class.getName());
log.info("拉取kafka消息配置:{}", props);
try (KafkaConsumer<String, String> consumer = new KafkaConsumer<>(props)) {
TopicPartition topicPartition = new TopicPartition(topic, partition);
consumer.assign(Collections.singletonList(topicPartition));
consumer.seek(topicPartition, offset);
ConsumerRecords<String, String> consumerRecords = consumer.poll(Duration.ofSeconds(DURATION_TIMEOUT));
if (consumerRecords.count() == COUNT) {
Iterator<ConsumerRecord<String, String>> iterator = consumerRecords.iterator();
ConsumerRecord<String, String> consumerRecord = iterator.next();
log.info("拉取kafka消息结果,topic:{},partition:{},offset:{},msg:{}", consumerRecord.topic(), consumerRecord.partition(), consumerRecord.offset(), consumerRecord.value());
return consumerRecord;
} else {
throw new KafkaException(ReturnCode.DATA_STATUS_ERROR, "拉取kafka消息数量错误");
}
} catch (Exception e) {
throw new KafkaException(ReturnCode.FAIL, "拉取kafka消息错误");
}
}
}