当消费者发生反序列化失败时会导致消费者偏移量不会向后移动,而且海量的错误日志会将磁盘写满,因此需要针对此类错误进行手动处理
ConcurrentKafkaListenerContainerFactory<String, MyObject> factory =
new ConcurrentKafkaListenerContainerFactory<>();
factory.setConsumerFactory(consumerFactory);
factory.setCommonErrorHandler(new CommonErrorHandler(){
@Override
public void handleOtherException(Exception thrownException, Consumer<?, ?> consumer, MessageListenerContainer container, boolean batchListener) {
String s = thrownException.getMessage().split("Error deserializing key/value for partition ")[1].split(". If needed, please seek past the record to continue consumption.")[0];
String topics = s.split("-")[0];
int offset = Integer.parseInt(s.split("offset ")[1]);
int partition = Integer.parseInt(s.split("-")[1].split(" at")[0]);
TopicPartition topicPartition = new TopicPartition(topics, partition);
consumer.seek(topicPartition, offset + 1);
consumer.commitAsync();
LOG.error("Skipping " + topics + "-" + partition + " offset " + offset);
}
});