组管理器是管理多个GroupMetadata
一.核心字段
其中最核心的就是groupMetadataCache ,底层就是一个ConcurrentHashMap,消费组名,对应GroupMetadata
private val groupMetadataCache = new Pool[String, GroupMetadata]
二.消费者组管理
核心方法就是对groupMetadataCache操作
消费者组元数据管理分为查询获取组信息、添加组、移除组和加载组信息
方法分析
1.持久化Group信息storeGroup
def storeGroup(group: GroupMetadata,
groupAssignment: Map[String, Array[Byte]],
responseCallback: Errors => Unit): Unit = {
getMagic(partitionFor(group.groupId)) match {
case Some(magicValue) =>
//生成groupMetadata消息开始
val timestampType = TimestampType.CREATE_TIME
val timestamp = time.milliseconds()
val key = GroupMetadataManager.groupMetadataKey(group.groupId)
val value = GroupMetadataManager.groupMetadataValue(group, groupAssignment, interBrokerProtocolVersion)
val records = {
val buffer = ByteBuffer.allocate(AbstractRecords.estimateSizeInBytes(magicValue, compressionType,
Seq(new SimpleRecord(timestamp, key, value)).asJava))
val builder = MemoryRecords.builder(buffer, magicValue, compressionType, timestampType, 0L)
builder.append(timestamp, key, value)
builder.build()
}
//生成groupMetadata消息结束
//找到要保存的__consumer_offsets的分区
val groupMetadataPartition = new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, partitionFor(group.groupId))
val groupMetadataRecords = Map(groupMetadataPartition -> records)
val generationId = group.generationId
//处理错误码
def putCacheCallback(responseStatus: Map[TopicPartition, PartitionResponse]): Unit = {
........
}
//向__consumer_offsets写入groupMetadata record
appendForGroup(group, groupMetadataRecords, putCacheCallback)
case None =>
responseCallback(Errors.NOT_COORDINATOR)
None
}
}
2.接管或新创建Group Manager的时候,需要从__consumer_offsets里取出需要的group metadata和offsets matadata信息:doLoadGroupsAndOffsets
private def doLoadGroupsAndOffsets(topicPartition: TopicPartition, onGroupLoaded: GroupMetadata => Unit): Unit = { def logEndOffset: Long = replicaManager.getLogEndOffset(topicPartition).getOrElse(-1L) replicaManager.getLog(topicPartition) match { case Some(log) => val loadedOffsets = mutable.Map[GroupTopicPartition, CommitRecordMetadataAndOffset]() val pendingOffsets = mutable.Map[Long, mutable.Map[GroupTopicPartition, CommitRecordMetadataAndOffset]]() val loadedGroups = mutable.Map[String, GroupMetadata]() val removedGroups = mutable.Set[String]() // buffer may not be needed if records are read from memory var buffer = ByteBuffer.allocate(0) // loop breaks if leader changes at any time during the load, since logEndOffset is -1 var currOffset = log.logStartOffset // loop breaks if no records have been read, since the end of the log has been reached var readAtLeastOneRecord = true while (currOffset < logEndOffset && readAtLeastOneRecord && !shuttingDown.get()) { //从log读取一些消息(__consumer_offsets里读消息) val fetchDataInfo = log.read(currOffset, maxLength = config.loadBufferSize, isolation = FetchLogEnd, minOneMessage = true) readAtLeastOneRecord = fetchDataInfo.records.sizeInBytes > 0 //把消息转化为MemoryRecords val memRecords = fetchDataInfo.records match { case records: MemoryRecords => records case fileRecords: FileRecords => val sizeInBytes = fileRecords.sizeInBytes val bytesNeeded = Math.max(config.loadBufferSize, sizeInBytes) // minOneMessage = true in the above log.read means that the buffer may need to be grown to ensure progress can be made if (buffer.capacity < bytesNeeded) { if (config.loadBufferSize < bytesNeeded) warn(s"Loaded offsets and group metadata from $topicPartition with buffer larger ($bytesNeeded bytes) than " + s"configured offsets.load.buffer.size (${config.loadBufferSize} bytes)") buffer = ByteBuffer.allocate(bytesNeeded) } else { buffer.clear() } fileRecords.readInto(buffer, 0) MemoryRecords.readableRecords(buffer) } //解析MemoryRecords memRecords.batches.forEach { batch => val isTxnOffsetCommit = batch.isTransactional //事务相关的控制消息的处理,暂时不解读 if (batch.isControlBatch) { ...... } else { var batchBaseOffset: Option[Long] = None for (record <- batch.asScala) { require(record.hasKey, "Group metadata/offset entry key should not be null") if (batchBaseOffset.isEmpty) batchBaseOffset = Some(record.offset) //判断消息的key,OffsetKey和groupMetadataKey两种 GroupMetadataManager.readMessageKey(record.key) match { //处理offsets消息 case offsetKey: OffsetKey => // load offset val groupTopicPartition = offsetKey.key //没有value说明offset被删除了 if (!record.hasValue) { loadedOffsets.remove(groupTopicPartition) } else { //有value取出offset保存起来 val offsetAndMetadata = GroupMetadataManager.readOffsetMessageValue(record.value) //CommitRecordMetadataAndOffset保存了__cosumer_offsets 的offset和group消费的topic的offset的对应关系 loadedOffsets.put(groupTopicPartition, CommitRecordMetadataAndOffset(batchBaseOffset, offsetAndMetadata)) } //处理group metadata消息 case groupMetadataKey: GroupMetadataKey => // load group metadata val groupId = groupMetadataKey.key val groupMetadata = GroupMetadataManager.readGroupMessageValue(groupId, record.value, time) //根据是否有value保存到loaded或者remove的map里 if (groupMetadata != null) { removedGroups.remove(groupId) loadedGroups.put(groupId, groupMetadata) } else { loadedGroups.remove(groupId) removedGroups.add(groupId) } case unknownKey => throw new IllegalStateException(s"Unexpected message key $unknownKey while loading offsets and group metadata") } } } currOffset = batch.nextOffset } } val (groupOffsets, emptyGroupOffsets) = loadedOffsets // 解释:Map[GroupTopicPartition, CommitRecordMetadataAndOffset] .groupBy(_._1.group) //解释:Map[group,Map[GroupTopicPartition, CommitRecordMetadataAndOffset]] .map { case (k, v) => //解释:(key=group,value=Map[GroupTopicPartition, CommitRecordMetadataAndOffset]) k -> v.map { case (groupTopicPartition, offset) => (groupTopicPartition.topicPartition, offset) } }.partition { case (group, _) => loadedGroups.contains(group) } //根据有offsets信息,有无group信息分组 val pendingOffsetsByGroup = ....事务相关,不解读 loadedGroups.values.foreach { group => //解释:Map[String, GroupMetadata] val offsets = groupOffsets.getOrElse(group.groupId, Map.empty[TopicPartition, CommitRecordMetadataAndOffset]) val pendingOffsets = pendingGroupOffsets.getOrElse(group.groupId, Map.empty[Long, mutable.Map[TopicPartition, CommitRecordMetadataAndOffset]]) loadGroup(group, offsets, pendingOffsets) //更新 offsets哪个map onGroupLoaded(group)//回调 } // load groups which store offsets in kafka, but which have no active members and thus no group // metadata stored in the log (emptyGroupOffsets.keySet ++ pendingEmptyGroupOffsets.keySet).foreach { groupId => val group = new GroupMetadata(groupId, Empty, time) val offsets = emptyGroupOffsets.getOrElse(groupId, Map.empty[TopicPartition, CommitRecordMetadataAndOffset]) val pendingOffsets = pendingEmptyGroupOffsets.getOrElse(groupId, Map.empty[Long, mutable.Map[TopicPartition, CommitRecordMetadataAndOffset]]) debug(s"Loaded group metadata $group with offsets $offsets and pending offsets $pendingOffsets") loadGroup(group, offsets, pendingOffsets) onGroupLoaded(group) } removedGroups.foreach { groupId => // if the cache already contains a group which should be removed, raise an error. Note that it // is possible (however unlikely) for a consumer group to be removed, and then to be used only for // offset storage (i.e. by "simple" consumers) if (groupMetadataCache.contains(groupId) && !emptyGroupOffsets.contains(groupId)) throw new IllegalStateException(s"Unexpected unload of active group $groupId while " + s"loading partition $topicPartition") } } }
三.消费者组位移管理
1.保存位移storeOffsets
和stroeGroup类似 生成offsets的record,写入到对应的__consumer_offsets的partition里,更新offsets的map
2.查询位移getOffsets
def getOffsets(groupId: String, requireStable: Boolean, topicPartitionsOpt: Option[Seq[TopicPartition]]): Map[TopicPartition, PartitionData] = { val group = groupMetadataCache.get(groupId) if (group == null) { ...返回错误 } else { group.inLock { if (group.is(Dead)) { ....返回错误 } else { val topicPartitions = topicPartitionsOpt.getOrElse(group.allOffsets.keySet) topicPartitions.map { topicPartition => if (requireStable && group.hasPendingOffsetCommitsForTopicPartition(topicPartition)) { ....返回错误 } else { val partitionData = group.offset(topicPartition) match { case None => ...返回错误 case Some(offsetAndMetadata) => new PartitionData(offsetAndMetadata.offset, offsetAndMetadata.leaderEpoch, offsetAndMetadata.metadata, Errors.NONE) } topicPartition -> partitionData } }.toMap } } } }