kafka监听topic消费_如何监控kafka消费Lag情况

/**

* 根据topic 获取offset,该结果是不同group 组不同patition的当前消费offset

*

* @param topic

* @return Map>>

* @throws InterruptedException

* @throws ExecutionException

* @throws TimeoutException

*/

public Map>> listConsumerGroupOffsets(String topic)

throws InterruptedException, ExecutionException, TimeoutException {

Set groupIds = this.listConsumerGroups(topic);

Map>> consumerGroupOffsets = new HashMap<>();

groupIds.forEach(groupId -> {

Set> consumerPatitionOffsets = new HashSet<>();

try {

consumerPatitionOffsets = this.adminClient.listConsumerGroupOffsets(groupId)

.partitionsToOffsetAndMetadata().get(30, TimeUnit.SECONDS).entrySet().stream()

.filter(entry -> topic.equalsIgnoreCase(entry.getKey().topic())).collect(Collectors.toSet());

;

} catch (InterruptedException e) {

e.printStackTrace();

} catch (ExecutionException e) {

e.printStackTrace();

} catch (TimeoutException e) {

e.printStackTrace();

}

consumerGroupOffsets.put(groupId, consumerPatitionOffsets);

});

return consumerGroupOffsets;

}

/**

* 根据topic 获取topicConsumerGroupStates,其中未包含lag/logEndOffset(consumer可见offset)

*

* @param topic

* @return

* @throws InterruptedException

* @throws ExecutionException

* @throws TimeoutException

*/

public List describeConsumerGroups(String topic)

throws InterruptedException, ExecutionException, TimeoutException {

final List topicConsumerGroupStates = new ArrayList<>();

Set groupIds = this.listConsumerGroups(topic);

Map groupDetails = this.adminClient.describeConsumerGroups(groupIds).all()

.get(30, TimeUnit.SECONDS);

Map>> consumerPatitionOffsetMap = this

.listConsumerGroupOffsets(topic);

groupDetails.entrySet().forEach(entry -> {

String groupId = entry.getKey();

ConsumerGroupDescription description = entry.getValue();

TopicConsumerGroupState topicConsumerGroupState = new TopicConsumerGroupState();

topicConsumerGroupState.setGroupId(groupId);

topicConsumerGroupState.setConsumerMethod("broker");

topicConsumerGroupState.setConsumerGroupState(description.state());

// 获取group下不同patition消费offset信息

Set> consumerPatitionOffsets = consumerPatitionOffsetMap

.get(groupId);

List partitionAssignmentStates = new ArrayList<>();

if (!description.members().isEmpty()) {

// 获取存在consumer(memeber存在的情况)

partitionAssignmentStates = this.withMembers(consumerPatitionOffsets, topic, groupId, description);

} else {

// 获取不存在consumer

partitionAssignmentStates = this.withNoMembers(consumerPatitionOffsets, topic, groupId);

}

topicConsumerGroupState.setPartitionAssignmentStates(partitionAssignmentStates);

topicConsumerGroupStates.add(topicConsumerGroupState);

});

return topicConsumerGroupStates;

}

private List withMembers(

Set> consumerPatitionOffsets, String topic, String groupId,

ConsumerGroupDescription description) {

List partitionAssignmentStates = new ArrayList<>();

Map consumerPatitionOffsetMap = new HashMap<>();

consumerPatitionOffsets.forEach(entryInfo -> {

TopicPartition topicPartition = entryInfo.getKey();

OffsetAndMetadata offsetAndMetadata = entryInfo.getValue();

consumerPatitionOffsetMap.put(topicPartition.partition(), offsetAndMetadata.offset());

});

description.members().forEach(memberDescription -> {

memberDescription.assignment().topicPartitions().forEach(topicPation -> {

PartitionAssignmentState partitionAssignmentState = new PartitionAssignmentState();

partitionAssignmentState.setPartition(topicPation.partition());

partitionAssignmentState.setTopic(topic);

partitionAssignmentState.setClientId(memberDescription.clientId());

partitionAssignmentState.setGroup(groupId);

partitionAssignmentState.setConsumerId(memberDescription.consumerId());

partitionAssignmentState.setHost(memberDescription.host());

partitionAssignmentState.setOffset(consumerPatitionOffsetMap.get(topicPation.partition()));

partitionAssignmentStates.add(partitionAssignmentState);

});

});

return partitionAssignmentStates;

}

private List withNoMembers(

Set> consumerPatitionOffsets, String topic, String groupId) {

List partitionAssignmentStates = new ArrayList<>();

consumerPatitionOffsets.forEach(entryInfo -> {

TopicPartition topicPartition = entryInfo.getKey();

OffsetAndMetadata offsetAndMetadata = entryInfo.getValue();

PartitionAssignmentState partitionAssignmentState = new PartitionAssignmentState();

partitionAssignmentState.setPartition(topicPartition.partition());

partitionAssignmentState.setTopic(topic);

partitionAssignmentState.setGroup(groupId);

partitionAssignmentState.setOffset(offsetAndMetadata.offset());

partitionAssignmentStates.add(partitionAssignmentState);

});

return partitionAssignmentStates;

}