kafka日志存储（二）：ByteBufferMessageSet

使用ByteBuffer保存消息数据，提供以下三个功能。
1. 把Message集合按照指定的压缩类型进行压缩，构建ByteBufferMessageSet对象
2. 提供迭代器。
3. 提供消息验证和offset分配功能。

private def create(offsetAssigner: OffsetAssigner,
                 compressionCodec: CompressionCodec,
                 wrapperMessageTimestamp: Option[Long],
                 timestampType: TimestampType,
                 messages: Message*): ByteBuffer = {
    // 集合为空，则返回空的buffer对象
    if (messages.isEmpty)
      MessageSet.Empty.buffer
    // 不需要对message集合进行压缩
    else if (compressionCodec == NoCompressionCodec) {
      val buffer = ByteBuffer.allocate(MessageSet.messageSetSize(messages))
      //为每个消息分配offset，写入buffer
      for (message <- messages) writeMessage(buffer, message, offsetAssigner.nextAbsoluteOffset())
      buffer.rewind()
      buffer
    } else {
        //需要对消息进行压缩
        val magicAndTimestamp = wrapperMessageTimestamp match {
        case Some(ts) => MagicAndTimestamp(messages.head.magic, ts)
        case None => MessageSet.magicAndLargestTimestamp(messages)
      }
      var offset = -1L
      val messageWriter = new MessageWriter(math.min(math.max(MessageSet.messageSetSize(messages) / 2, 1024), 1 << 16))
      messageWriter.write(codec = compressionCodec, timestamp = magicAndTimestamp.timestamp, timestampType = timestampType, magicValue = magicAndTimestamp.magic) { 
        outputStream => // 创建压缩类型的输出流
        val output = new DataOutputStream(CompressionFactory(compressionCodec, magicAndTimestamp.magic, outputStream))
        try {
          for (message <- messages) {
              //遍历写入内层压缩信息。
            offset = offsetAssigner.nextAbsoluteOffset()
            if (message.magic != magicAndTimestamp.magic)
              throw new IllegalArgumentException("Messages in the message set must have same magic value")
            // Use inner offset if magic value is greater than 0
            if (magicAndTimestamp.magic > Message.MagicValue_V0)
              output.writeLong(offsetAssigner.toInnerOffset(offset))
            else
              output.writeLong(offset)
            output.writeInt(message.size)
            output.write(message.buffer.array, message.buffer.arrayOffset, message.buffer.limit)
          }
        } finally {
          output.close()
        }
      }
      val buffer = ByteBuffer.allocate(messageWriter.size + MessageSet.LogOverhead)
      //按照消息格式写入整个外层信息，注意，外层信息的offset是最后一条内层信息。
      writeMessage(buffer, messageWriter, offset)
      buffer.rewind()
      buffer
    }
}

ByteBufferMessageSet.validateMessagesAndAssignOffsets()方法实现了验证消息并分配offset的功能

private[kafka] def validateMessagesAndAssignOffsets(offsetCounter: LongRef,//分配offset的起始值
                                                      now: Long,
                                                      sourceCodec: CompressionCodec,
                                                      targetCodec: CompressionCodec,
                                                      compactedTopic: Boolean = false,
                                                      messageFormatVersion: Byte = Message.CurrentMagicValue,
                                                      messageTimestampType: TimestampType,
                                                      messageTimestampDiffMaxMs: Long): (ByteBufferMessageSet, Boolean) = {
    if (sourceCodec == NoCompressionCodec && targetCodec == NoCompressionCodec) {
      // 检测所有的magic是否和指定magic一致
      if (!isMagicValueInAllWrapperMessages(messageFormatVersion)) {
        // 不一致需要统一，需要创建新的ByteBufferMessageSet，长度可能会发送改变还进行offset的分配，验证并且更新offset
        (convertNonCompressedMessages(offsetCounter, compactedTopic, now, messageTimestampType, messageTimestampDiffMaxMs,
          messageFormatVersion), true)
      } else {
        // 非压缩消息且magic统一。
        (validateNonCompressedMessagesAndAssignOffsetInPlace(offsetCounter, now, compactedTopic, messageTimestampType,
          messageTimestampDiffMaxMs), false)
      }
    } else {
      // Deal with compressed messages
      // We cannot do in place assignment in one of the following situations:
      // 1. Source and target compression codec are different
      // 2. When magic value to use is 0 because offsets need to be overwritten
      // 3. When magic value to use is above 0, but some fields of inner messages need to be overwritten.
      // 4. Message format conversion is needed.

      // 检查1：消息压缩的类型和此Brokers指定的压缩类型不一致，需要新压缩。/
      
      var inPlaceAssignment = sourceCodec == targetCodec && 
          //检查1：magic为零
          messageFormatVersion > Message.MagicValue_V0

      var maxTimestamp = Message.NoTimestamp
      val expectedInnerOffset = new LongRef(0)
      val validatedMessages = new mutable.ArrayBuffer[Message]
      //遍历内层压缩信息，此步骤会解压
      this.internalIterator(isShallow = false).foreach { messageAndOffset =>
        val message = messageAndOffset.message
        validateMessageKey(message, compactedTopic)

        if (message.magic > Message.MagicValue_V0 && messageFormatVersion > Message.MagicValue_V0) {
          // 检查3，检查时间戳
          validateTimestamp(message, now, messageTimestampType, messageTimestampDiffMaxMs)
          // 检查内部offset是否正常
          if (messageAndOffset.offset != expectedInnerOffset.getAndIncrement())
            inPlaceAssignment = false
          maxTimestamp = math.max(maxTimestamp, message.timestamp)
        }

        if (sourceCodec != NoCompressionCodec && message.compressionCodec != NoCompressionCodec)
          throw new InvalidMessageException("Compressed outer message should not have an inner message with a " +
            s"compression attribute set: $message")

        // 检查4：消息格式需要转换
        if (message.magic != messageFormatVersion)
          inPlaceAssignment = false

        validatedMessages += message.toFormatVersion(messageFormatVersion)
      }
        //不满足条件，不能复用当前ByteBufferMessageSet的场景
      if (!inPlaceAssignment) {
        val wrapperMessageTimestamp = {
          if (messageFormatVersion == Message.MagicValue_V0)
            Some(Message.NoTimestamp)
          else if (messageFormatVersion > Message.MagicValue_V0 && messageTimestampType == TimestampType.CREATE_TIME)
            Some(maxTimestamp)
          else // Log append time
            Some(now)
        }

        // 创建新的并压缩
        (new ByteBufferMessageSet(compressionCodec = targetCodec,
                                  offsetCounter = offsetCounter,
                                  wrapperMessageTimestamp = wrapperMessageTimestamp,
                                  timestampType = messageTimestampType,
                                  messages = validatedMessages: _*), true)
      } else {
        // 复用.ByteBufferMessageSet。不需要重新压缩
        buffer.putLong(0, offsetCounter.addAndGet(validatedMessages.size) - 1)
        // validate the messages
        validatedMessages.foreach(_.ensureValid())

        var crcUpdateNeeded = true
        val timestampOffset = MessageSet.LogOverhead + Message.TimestampOffset
        val attributeOffset = MessageSet.LogOverhead + Message.AttributesOffset
        val timestamp = buffer.getLong(timestampOffset)
        val attributes = buffer.get(attributeOffset)
        if (messageTimestampType == TimestampType.CREATE_TIME && timestamp == maxTimestamp)
          // We don't need to recompute crc if the timestamp is not updated.
          crcUpdateNeeded = false
        else if (messageTimestampType == TimestampType.LOG_APPEND_TIME) {
          // 更新外层消息的offset，为内部最后一条压缩消息的offset
          buffer.putLong(timestampOffset, now)
          buffer.put(attributeOffset, messageTimestampType.updateAttributes(attributes))
        }

        if (crcUpdateNeeded) {
          // 更新外层信息的时间戳，和crc32
          buffer.position(MessageSet.LogOverhead)
          val wrapperMessage = new Message(buffer.slice())
          Utils.writeUnsignedInt(buffer, MessageSet.LogOverhead + Message.CrcOffset, wrapperMessage.computeChecksum)
        }
        buffer.rewind()
        (this, false)
      }
    }
  }