// Rebuild producer state until lastOffset. This method may be called from the recovery code path, and thus must be // free of all side-effects, i.e. it must not update any log-specific state. private def rebuildProducerState(lastOffset: Long, reloadFromCleanShutdown: Boolean, producerStateManager: ProducerStateManager): Unit = lock synchronized { checkIfMemoryMappedBufferClosed() val messageFormatVersion = config.messageFormatVersion.recordVersion.value val segments = logSegments val offsetsToSnapshot = if (segments.nonEmpty) { val nextLatestSegmentBaseOffset = lowerSegment(segments.last.baseOffset).map(_.baseOffset) Seq(nextLatestSegmentBaseOffset, Some(segments.last.baseOffset), Some(lastOffset)) } else { Seq(Some(lastOffset)) } info(s "Loading producer state till offset $lastOffset with message format version $messageFormatVersion" ) // We want to avoid unnecessary scanning of the log to build the producer state when the broker is being // upgraded. The basic idea is to use the absence of producer snapshot files to detect the upgrade case, // but we have to be careful not to assume too much in the presence of broker failures. The two most common // upgrade cases in which we expect to find no snapshots are the following: // // 1. The broker has been upgraded, but the topic is still on the old message format. // 2. The broker has been upgraded, the topic is on the new message format, and we had a clean shutdown. // // If we hit either of these cases, we skip producer state loading and write a new snapshot at the log end // offset (see below). The next time the log is reloaded, we will load producer state using this snapshot // (or later snapshots). Otherwise, if there is no snapshot file, then we have to rebuild producer state // from the first segment. //我们希望避免在代理运行时不必要地扫描日志以构建生产者状态升级 //基本思想是利用生产者快照文件的缺失来检测升级案例,但是我们必须小心,不要在代理失败的情况下做太多假设 //最常见的两种升级的情况下,我们希望找到没有快照如下: //1.代理已经升级,但是主题仍然是关于旧的消息格式 //2.代理已经升级,主题是关于新的消息格式,并且我们已经干净地关闭了 //如果遇到上述任何一种情况,我们将跳过生产者状态加载,并在日志端写入一个新的快照偏移量(见下) //下次重新加载日志时,我们将使用此快照加载生产者状态(或以后的快照) //否则,如果没有快照文件,则必须重新构建生产者状态从第一个片段。 if (messageFormatVersion < RecordBatch.MAGIC_VALUE_V2 || (producerStateManager.latestSnapshotOffset.isEmpty && reloadFromCleanShutdown)) { // To avoid an expensive scan through all of the segments, we take empty snapshots from the start of the // last two segments and the last offset. This should avoid the full scan in the case that the log needs // truncation. offsetsToSnapshot.flatten.foreach { offset => producerStateManager.updateMapEndOffset(offset) producerStateManager.takeSnapshot() } } else { val isEmptyBeforeTruncation = producerStateManager.isEmpty && producerStateManager.mapEndOffset >= lastOffset producerStateManager.truncateAndReload(logStartOffset, lastOffset, time.milliseconds()) // Only do the potentially expensive reloading if the last snapshot offset is lower than the log end // offset (which would be the case on first startup) and there were active producers prior to truncation // (which could be the case if truncating after initial loading). If there weren't, then truncating // shouldn't change that fact (although it could cause a producerId to expire earlier than expected), // and we can skip the loading. This is an optimization for users which are not yet using // idempotent/transactional features yet. if (lastOffset > producerStateManager.mapEndOffset && !isEmptyBeforeTruncation) { val segmentOfLastOffset = floorLogSegment(lastOffset) logSegments(producerStateManager.mapEndOffset, lastOffset).foreach { segment => val startOffset = Utils.max(segment.baseOffset, producerStateManager.mapEndOffset, logStartOffset) producerStateManager.updateMapEndOffset(startOffset) if (offsetsToSnapshot.contains(Some(segment.baseOffset))) producerStateManager.takeSnapshot() val maxPosition = if (segmentOfLastOffset.contains(segment)) { Option(segment.translateOffset(lastOffset)) .map(_.position) .getOrElse(segment.size) } else { segment.size } val fetchDataInfo = segment.read(startOffset, maxSize = Int.MaxValue, maxPosition = maxPosition, minOneMessage = false ) if (fetchDataInfo != null ) loadProducersFromLog(producerStateManager, fetchDataInfo.records) } } producerStateManager.updateMapEndOffset(lastOffset) producerStateManager.takeSnapshot() } } |