Kafka源码分析（3）

三 API Layer

1、KafkaApis

         该类是各种API的封装，通过传入的requestId来决定调用何种API，最重要的handle()方法如下所示：

def handle(request: RequestChannel.Request) {
    try{
      trace("Handling request: " + request.requestObj + " from client: " + request.remoteAddress)
      request.requestId match {
        case RequestKeys.ProduceKey => handleProducerOrOffsetCommitRequest(request)
        case RequestKeys.FetchKey => handleFetchRequest(request)
        case RequestKeys.OffsetsKey => handleOffsetRequest(request)
        case RequestKeys.MetadataKey => handleTopicMetadataRequest(request)
        case RequestKeys.LeaderAndIsrKey => handleLeaderAndIsrRequest(request)
        case RequestKeys.StopReplicaKey => handleStopReplicaRequest(request)
        case RequestKeys.UpdateMetadataKey => handleUpdateMetadataRequest(request)
        case RequestKeys.ControlledShutdownKey => handleControlledShutdownRequest(request)
        case RequestKeys.OffsetCommitKey => handleOffsetCommitRequest(request)
        case RequestKeys.OffsetFetchKey => handleOffsetFetchRequest(request)
        case RequestKeys.ConsumerMetadataKey => handleConsumerMetadataRequest(request)
        case requestId => throw new KafkaException("Unknown api code " + requestId)
      }
    } catch {
      case e: Throwable =>
        request.requestObj.handleError(e, requestChannel, request)
        error("error when handling request %s".format(request.requestObj), e)
    } finally
      request.apiLocalCompleteTimeMs = SystemTime.milliseconds
  }

         在这个版本中，handleProducerRequest和handleOffsetCommitRequest两个方法被合并成handleProducerOrOffsetCommitRequest一个方法（0.8以前是分开的），因为producer生成消息后也要进行offsetcommit，所以两个操作的绝大多数代码是相似的。从以下代码中可见，kafka先根据request.requestId来封装request对象，然后根据produceRequest.requiredAcks的值（代表该request是否需要确认，即同步还是异步的）来处理数据和生成返回值：

	def handleProducerOrOffsetCommitRequest(request: RequestChannel.Request) {
    val (produceRequest, offsetCommitRequestOpt) =
      if (request.requestId == RequestKeys.OffsetCommitKey) {
        val offsetCommitRequest = request.requestObj.asInstanceOf[OffsetCommitRequest]
        OffsetCommitRequest.changeInvalidTimeToCurrentTime(offsetCommitRequest)
        (producerRequestFromOffsetCommit(offsetCommitRequest), Some(offsetCommitRequest))
      } else {
        (request.requestObj.asInstanceOf[ProducerRequest], None)
      }

    if (produceRequest.requiredAcks > 1 || produceRequest.requiredAcks < -1) {
      warn(("Client %s from %s sent a produce request with request.required.acks of %d, which is now deprecated and will " +
            "be removed in next release. Valid values are -1, 0 or 1. Please consult Kafka documentation for supported " +
            "and recommended configuration.").format(produceRequest.clientId, request.remoteAddress, produceRequest.requiredAcks))
    }

    val sTime = SystemTime.milliseconds
    val localProduceResults = appendToLocalLog(produceRequest, offsetCommitRequestOpt.nonEmpty)
    debug("Produce to local log in %d ms".format(SystemTime.milliseconds - sTime))

    val firstErrorCode = localProduceResults.find(_.errorCode != ErrorMapping.NoError).map(_.errorCode).getOrElse(ErrorMapping.NoError)

    val numPartitionsInError = localProduceResults.count(_.error.isDefined)
    if(produceRequest.requiredAcks == 0) {
      // no operation needed if producer request.required.acks = 0; however, if there is any exception in handling the request, since
      // no response is expected by the producer the handler will send a close connection response to the socket server
      // to close the socket so that the producer client will know that some exception has happened and will refresh its metadata
      if (numPartitionsInError != 0) {
        info(("Send the close connection response due to error handling produce request " +
          "[clientId = %s, correlationId = %s, topicAndPartition = %s] with Ack=0")
          .format(produceRequest.clientId, produceRequest.correlationId, produceRequest.topicPartitionMessageSizeMap.keySet.mkString(",")))
        requestChannel.closeConnection(request.processor, request)
      } else {

        if (firstErrorCode == ErrorMapping.NoError)
          offsetCommitRequestOpt.foreach(ocr => offsetManager.putOffsets(ocr.groupId, ocr.requestInfo))

        if (offsetCommitRequestOpt.isDefined) {
          val response = offsetCommitRequestOpt.get.responseFor(firstErrorCode, config.offsetMetadataMaxSize)
          requestChannel.sendResponse(new RequestChannel.Response(request, new BoundedByteBufferSend(response)))
        } else
          requestChannel.noOperation(request.processor, request)
      }
    } else if (produceRequest.requiredAcks == 1 ||
        produceRequest.numPartitions <= 0 ||
        numPartitionsInError == produceRequest.numPartitions) {

      if (firstErrorCode == ErrorMapping.NoError) {
        offsetCommitRequestOpt.foreach(ocr => offsetManager.putOffsets(ocr.groupId, ocr.requestInfo) )
      }

      val statuses = localProduceResults.map(r => r.key -> ProducerResponseStatus(r.errorCode, r.start)).toMap
      val response = offsetCommitRequestOpt.map(_.responseFor(firstErrorCode, config.offsetMetadataMaxSize))
                                           .getOrElse(ProducerResponse(produceRequest.correlationId, statuses))

      requestChannel.sendResponse(new RequestChannel.Response(request, new BoundedByteBufferSend(response)))
    } else {
      // create a list of (topic, partition) pairs to use as keys for this delayed request
      val producerRequestKeys = produceRequest.data.keys.toSeq
      val statuses = localProduceResults.map(r =>
        r.key -> DelayedProduceResponseStatus(r.end + 1, ProducerResponseStatus(r.errorCode, r.start))).toMap
      val delayedRequest =  new DelayedProduce(
        producerRequestKeys,
        request,
        produceRequest.ackTimeoutMs.toLong,
        produceRequest,
        statuses,
        offsetCommitRequestOpt)

      // add the produce request for watch if it's not satisfied, otherwise send the response back
      val satisfiedByMe = producerRequestPurgatory.checkAndMaybeWatch(delayedRequest)
      if (satisfiedByMe)
        producerRequestPurgatory.respond(delayedRequest)
    }

    // we do not need the data anymore
    produceRequest.emptyData()
  }

         另一个值得注意的方法是handleOffsetCommitRequest，一个消费者在消费完某个partition的数据后，会自动将offset提交（当然也可以手动调用提交）。但这里有个问题是，如果consumer1和consumer2都在消费partition1的数据，consumer1先提交offset，此时consumer2 crash，这时候consumer2还没处理完的offset前的数据就丢失了，这是典型的at most once机制，也就是存在一定的丢失数据的风险。这个方法中判断offsetCommitRequest.versionId的值，如为0（老版本调用）则将offset值存入zookeeper中，如为1则调用上述的handleProducerOrOffsetCommitRequest方法，将offset值存入一个特定的topic中（OffsetManager类用来处理该工作）：

	def handleOffsetCommitRequest(request: RequestChannel.Request) {
    val offsetCommitRequest = request.requestObj.asInstanceOf[OffsetCommitRequest]
    if (offsetCommitRequest.versionId == 0) {
      // version 0 stores the offsets in ZK
      val responseInfo = offsetCommitRequest.requestInfo.map{
        case (topicAndPartition, metaAndError) => {
          val topicDirs = new ZKGroupTopicDirs(offsetCommitRequest.groupId, topicAndPartition.topic)
          try {
            ensureTopicExists(topicAndPartition.topic)
            if(metaAndError.metadata != null && metaAndError.metadata.length > config.offsetMetadataMaxSize) {
              (topicAndPartition, ErrorMapping.OffsetMetadataTooLargeCode)
            } else {
              ZkUtils.updatePersistentPath(zkClient, topicDirs.consumerOffsetDir + "/" +
                topicAndPartition.partition, metaAndError.offset.toString)
              (topicAndPartition, ErrorMapping.NoError)
            }
          } catch {
            case e: Throwable => (topicAndPartition, ErrorMapping.codeFor(e.getClass.asInstanceOf[Class[Throwable]]))
          }
        }
      }
      val response = new OffsetCommitResponse(responseInfo, offsetCommitRequest.correlationId)
      requestChannel.sendResponse(new RequestChannel.Response(request, new BoundedByteBufferSend(response)))
    } else {
      // version 1 and above store the offsets in a special Kafka topic
      handleProducerOrOffsetCommitRequest(request)
    }
  }

         其余方法和老版本中代码变化不大，也不详细展开了。

2、****Request/ ****Response

         ****可选的字符有OffsetCommit,LeaderAndIsr, StopReplica, UpdateMetadata等等。这些类是用来包装API层的各种请求和响应消息的，并不对消息进行实质性处理，所以里面的代码也很简单，基本都是些序列化/反序列化的工作。典型地，一个Request/Response下会有以下方法：

         readFrom()：从一个ByteBuffer中读取数据并构造一个对象

         writeTo()：将数据写入一个ByteBuffer

         sizeInBytes()：计算对象大小

         describe(),toString()：将消息转换为一种类json的string格式

         handleError()：Request中才有，用于在异常时生成一个特定的Response

         代码就不列了。

3、RequestPurgatory

         该类为ProducerRequestPurgatory和FetchRequestPurgatory两个类的父类。Purgatory的本意是“炼狱”，在kafka里实际上就是缓冲区的意思，这从这两个子类的头部注释也可以看出：The purgatory holding delayed producer requests/The purgatoryholding delayed fetch requests。

         首先开来看RequestPurgatory.scala文件，在其头部定义了一个DelayedRequest类，这个类继承了基类DelayedItem（我们不深究），还有一个原子的布尔值，保证对该值的赋值操作是原子的。而这个类的功能在头部注释页说得很清楚了，就是用来封装延迟的请求的：

/**
 * A request whose processing needs to be delayed for at most the given delayMs
 * The associated keys are used for bookeeping, and represent the "trigger" that causes this request to check if it is satisfied,
 * for example a key could be a (topic, partition) pair.
 */
class DelayedRequest(val keys: Seq[Any], val request: RequestChannel.Request, delayMs: Long) extends DelayedItem[RequestChannel.Request](request, delayMs) {
  val satisfied = new AtomicBoolean(false)
}

         然后是抽象类RequestPurgatory，里面重点方法有：

         isSatisfiedByMe()：具体方法，尝试设置该request的satisfied字段

         checkAndMaybeWatch()：具体方法，用于将可以满足的request置为满足，将不能满足的request加入观察队列。因为扫描观察队列的线程可能大于1个，这里用了两阶段扫描的技巧，可以看下代码中注释的描述。

// The cost of checkSatisfied() is typically proportional to the number of keys. Calling
    // checkSatisfied() for each key is going to be expensive if there are many keys. Instead,
    // we do the check in the following way. Call checkSatisfied(). If the request is not satisfied,
    // we just add the request to all keys. Then we call checkSatisfied() again. At this time, if
    // the request is still not satisfied, we are guaranteed that it won't miss any future triggering
    // events since the request is already on the watcher list for all keys. This does mean that
    // if the request is satisfied (by another thread) between the two checkSatisfied() calls, the
    // request is unnecessarily added for watch. However, this is a less severe issue since the
    // expire reaper will clean it up periodically.

         update()：具体方法，用于获取指定key的所有watcher的最新满足请求列表。

         checkSatisfied()：抽象方法，需要子类自己实现

         expire()：抽象方法，需要子类自己实现

4、ProducerRequestPurgatory/FetchRequestPurgatory

         ProducerRequestPurgatory类，两个抽象方法的实现如下：

/**
   * Check if a specified delayed fetch request is satisfied
   */
  def checkSatisfied(delayedProduce: DelayedProduce) = delayedProduce.isSatisfied(replicaManager)

  /**
   * When a delayed produce request expires answer it with possible time out error codes
   */
  def expire(delayedProduce: DelayedProduce) {
    debug("Expiring produce request %s.".format(delayedProduce.produce))
    for ((topicPartition, responseStatus) <- delayedProduce.partitionStatus if responseStatus.acksPending)
      recordDelayedProducerKeyExpired(topicPartition)
    respond(delayedProduce)
  }

         可见消息使用DelayedProduce进行特化，在这个类的头部说明了，这个类型的request在以下条件被满足：

         A.若broker不是leader，则返回错误

         B.若broker是leader，1，如果发生一个localError，则返回错误；2，否则，最少requiredAcks个备份将返回给此请求

         FetchRequestPurgatory类，两个抽象方法的实现如下：

/**
   * Check if a specified delayed fetch request is satisfied
   */
  def checkSatisfied(delayedFetch: DelayedFetch): Boolean = delayedFetch.isSatisfied(replicaManager)

  /**
   * When a delayed fetch request expires just answer it with whatever data is present
   */
  def expire(delayedFetch: DelayedFetch) {
    debug("Expiring fetch request %s.".format(delayedFetch.fetch))
    val fromFollower = delayedFetch.fetch.isFromFollower
    recordDelayedFetchExpired(fromFollower)
    respond(delayedFetch)
  }

         可见消息使用DelayedFetch进行特化，在这个类的头部说明了，这个类型的request在以下条件被满足：

         A.当前broker已不是fetch操作所需的某些partition的leader时，返回其他那些仍是leader的partition的数据

         B.当前broker不能识别fetch操作所需的某些partition，返回其他partition的数据

         C.fetch操作要求的offset不在log的最后一个段（segment，log是以一堆segment文件的形式存储的）内，需要返回该段的全部数据

         D.累计已fetch的byte数大于该FetchRequest设定的最小需要byte数，则返回可用的全部数据