kafka日志存储（一）：FileMessageSet

FileMessageSet
FileMessageSet管理日志上一篇博客中介绍的日志文件。主要字段有：

class FileMessageSet private[kafka](
                                    //磁盘上对应的日志文件
                                    @volatile var file: File,
                                    //jdk的FileChannel类型，读写对应的日志文件
                                    private[log] val channel: FileChannel,
                                    //FileMessageSet除了表示一个完整的日志文件，还可以表示日志的分片，start和end表示分片的起始位置和结束位置。
                                    private[log] val start: Int,
                                    private[log] val end: Int,
                                    //表示当前文件是否是分片
                                    isSlice: Boolean) extends MessageSet with Logging {
    //FileMessageSet的大小，单位是字节。如果FileMessageSet是日志文件的分片，则表示分片的大小。如果是完整的日志，则记录整个文件的大小。
    //可能存在多个handle线程并发向同一个分区写入信息，所以用AtomicInteger
    private val _size =
    if(isSlice)
      new AtomicInteger(end - start) // don't check the file size if this is just a slice view
    else
      new AtomicInteger(math.min(channel.size.toInt, end) - start)
  
    /* 如果不是分片，则更新position到文件末尾 */
    if (!isSlice)
        channel.position(math.min(channel.size.toInt, end))
    
    //构造函数：
    def this(file: File, fileAlreadyExists: Boolean, initFileSize: Int, preallocate: Boolean) =
      this(file,
        //如果使用了preallocate进行预分配空间，则end被初始化为0
        channel = FileMessageSet.openChannel(file, mutable = true, fileAlreadyExists, initFileSize, preallocate),
        start = 0,
        end = ( if ( !fileAlreadyExists && preallocate ) 0 else Int.MaxValue),
        isSlice = false)
        
    def openChannel(file: File, mutable: Boolean, fileAlreadyExists: Boolean = false, initFileSize: Int = 0, preallocate: Boolean = false): FileChannel = {
        if (mutable) {//FileChannel是否可写
          if (fileAlreadyExists)
            //文件已经存在
            new RandomAccessFile(file, "rw").getChannel()
          else {
            //进行文件预分配
            if (preallocate) {
              val randomAccessFile = new RandomAccessFile(file, "rw")
              randomAccessFile.setLength(initFileSize)
              randomAccessFile.getChannel()
            }
            else
              new RandomAccessFile(file, "rw").getChannel()
          }
        }
        else
          new FileInputStream(file).getChannel()//创建只读的FileChannel
  }
}

FileMessageSet.append()实现了写日志文件的功能，参数是ByteBufferMessagingSet：

class FileMessageSet{
  def append(messages: ByteBufferMessageSet) {
    val written = messages.writeFullyTo(channel)
    _size.getAndAdd(written)
  }
}
  /** Write the messages in this set to the given channel */
object ByteBufferMessageSet {
  def writeFullyTo(channel: GatheringByteChannel): Int = {
    buffer.mark()
    var written = 0
    while (written < sizeInBytes)
      //ByteBufferMessageSet中的数据全部写入文件。
      written += channel.write(buffer)
    buffer.reset()
    written
  }
}

查找指定消息的功能在searchFor方法中实现。从指定的startingPosition开始逐条遍历FileMessageSet中的消息。

def searchFor(targetOffset: Long, startingPosition: Int): OffsetPosition = {
    //其实位置
    var position = startingPosition
    //用于读取offset+size的ByteBuffer(大小为12)
    val buffer = ByteBuffer.allocate(MessageSet.LogOverhead)
    //当前FileMessageSet的大小
    val size = sizeInBytes()
    //从position开始逐条遍历
    while(position + MessageSet.LogOverhead < size) {
      //重置指针，准备读入数据
      buffer.rewind()
      //读取offset+size
      channel.read(buffer, position)
      if(buffer.hasRemaining)
        throw new IllegalStateException("Failed to read complete buffer for targetOffset %d startPosition %d in %s"
                                        .format(targetOffset, startingPosition, file.getAbsolutePath))
      //重置指针，准备从byteBuffer中读出数据
      buffer.rewind()
      val offset = buffer.getLong()
      //判断消息的offset是否服务退出条件
      if(offset >= targetOffset)
        // 符合条件，把offset对应的position封装成OffsetPosition返回。
        return OffsetPosition(offset, position)
      //没找到则获取消息的size，移动position，准备读取下个消息。
      val messageSize = buffer.getInt()
      if(messageSize < Message.MinMessageOverhead)
        throw new IllegalStateException("Invalid message size: " + messageSize)
      position += MessageSet.LogOverhead + messageSize
    }
    null
}

FileMessageSet的truncateTo方法，负责把日志文件截断到targetSize

def truncateTo(targetSize: Int): Int = {
    val originalSize = sizeInBytes
    //检测targetSize的有效性
    if(targetSize > originalSize || targetSize < 0)
      throw new KafkaException("Attempt to truncate log segment to " + targetSize + " bytes failed, " +
                               " size of this log segment is " + originalSize + " bytes.")
    if (targetSize < channel.size.toInt) {
      //裁剪文件，重置size
      channel.truncate(targetSize)
      channel.position(targetSize)
      _size.set(targetSize)
    }
    //返回裁剪的字节数
    originalSize - targetSize
}