1.主要逻辑:
private Map<TopicPartition, OffsetAndTimestamp> getTimestampOffset(KafkaConsumer<String, byte[]> kafkaConsumer, String topic, Date date){
Map<TopicPartition, Long> startMap = new HashMap<>();
List<PartitionInfo> partitionInfos = kafkaConsumer.partitionsFor(topic);
for (PartitionInfo partitionInfo : partitionInfos) {
int partition = partitionInfo.partition();
TopicPartition topicPartition = new TopicPartition(topic, partition);
startMap.put(topicPartition, date.getTime());
}
return kafkaConsumer.offsetsForTimes(startMap);
}
@SneakyThrows
private void execute(KafkaConsumer<String, byte[]> kafkaConsumer, Date startTime, Date endTime){
Map<TopicPartition, OffsetAndTimestamp> startOffsetMap = getTimestampOffset(kafkaConsumer, topic, startTime);
Map<TopicPartition, OffsetAndTimestamp> endOffsetMap = getTimestampOffset(kafkaConsumer, topic, endTime);
log.info("topic {}, start offset {}, end offset {}", topic, startOffsetMap, endOffsetMap);
List<PartitionInfo> partitionInfos = kafkaConsumer.partitionsFor(topic);
List<TopicPartition> partitionList = new ArrayList<>();
for (PartitionInfo partitionInfo : partitionInfos) {
partitionList.add(new TopicPartition(topic, partitionInfo.partition()));
}
Map<Integer, Long> partitionMap;
kafkaConsumer.assign(partitionList);
for (Map.Entry<TopicPartition, OffsetAndTimestamp> entry : startOffsetMap.entrySet()) {
kafkaConsumer.seek(entry.getKey(), entry.getValue().offset());
}
Set<Integer> partitionSet;
if (endOffsetMap != null){
partitionMap = endOffsetMap.entrySet().stream().collect(Collectors.toMap(topicPartitionOffsetAndTimestampEntry -> topicPartitionOffsetAndTimestampEntry.getKey().partition(), topicPartitionOffsetAndTimestampEntry -> {
OffsetAndTimestamp value = topicPartitionOffsetAndTimestampEntry.getValue();
if (value != null){
return value.offset();
}else{
return Long.MAX_VALUE;
}
}, (t1, t2) -> t1));
partitionSet = partitionMap.entrySet().stream().map(Map.Entry::getKey).collect(Collectors.toSet());
}else{
partitionMap = Collections.emptyMap();
partitionSet = Collections.emptySet();
}
while(true){
Map<Integer, Long> partitionOffsetMap = new HashMap<>();
ConsumerRecords<String, byte[]> consumerRecords = kafkaConsumer.poll(Duration.ofMillis(10000));
for (ConsumerRecord<String, byte[]> consumerRecord : consumerRecords) {
partitionOffsetMap.compute(consumerRecord.partition(), (integer, aLong) -> aLong == null ? consumerRecord.offset() : Math.max(aLong, consumerRecord.offset()));
long timestamp = consumerRecord.timestamp();
if (timestamp < startTime.getTime() || timestamp > endTime.getTime()){
continue;
}
byte[] value = consumerRecord.value();
//todo 消费逻辑
}
for (Map.Entry<Integer, Long> entry : partitionOffsetMap.entrySet()) {
Integer partition = entry.getKey();
Long maxOffset = entry.getValue();
Long endOffset = partitionMap.get(partition);
if (endOffset != null && maxOffset > endOffset && !partitionSet.isEmpty()){
partitionSet.remove(partition);
}
}
if (consumerRecords.isEmpty() || (!partitionMap.isEmpty() && partitionSet.isEmpty())){
break;
}
}
}
2.调用逻辑
public void run(){
//添加kafka配置
Map<String, Object> kafkaConfig = new HashMap<>();
KafkaConsumer<String, byte[]> kafkaConsumer = new KafkaConsumer<>(kafkaConfig);
execute(kafkaConsumer, startTime, endTime);
}
3. 说明
在自定义kafka查询时(配合seek使用),需用kafkaConsumer.assign,而不是kafkaConsumer.subscribe