DelayedOperationPurgatory机制（五）：DelayProduce实现

DelayedProduce的字段分析如下：

class DelayedProduce(
                     // DelayedProduce的延迟时长
                     delayMs: Long,
                     // ProduceMetadata为一个ProducerRequest中所有相关分区记录了一些追加消息后的返回结果，主要用于判断DelayedProduce是否满足条件
                     produceMetadata: ProduceMetadata,
                     replicaManager: ReplicaManager,
                     // 满足条件或者到期执行时，在onComplete方法中调用回调函数，主要功能是向RequestChannels中对应responseQueue添加ProduceStatus字段，然后Processor线程发送给客户端
                     responseCallback: Map[TopicPartition, PartitionResponse] => Unit)
  extends DelayedOperation(delayMs) {
      // 在DlayedProduce初始化时，首先会对produceMetadata字段中的produceStatus集合进行设置
        produceMetadata.produceStatus.foreach { case (topicPartition, status) =>
    if (status.responseStatus.errorCode == Errors.NONE.code) {
      // Timeout error state will be cleared when required acks are received
      // 对应分区写入成功，等待ISR集合中副本完成同步
      // 如果写入异常，就不等待
      status.acksPending = true
      // 预设错误码，如果ISR集合在请求超时之前完成了同步，就清除这个错误码
      status.responseStatus.errorCode = Errors.REQUEST_TIMED_OUT.code
    } else {
        // 如果追加日志时已经跑出异常，就不用等这个分区对应的ISR返回ACK了
      status.acksPending = false
    }
  }

DelayedProduce实现了DelayedOperation.tryComplete方法，其主要逻辑是检测是否满足DelayedProduce的条件，并在满足执行条件时调用forceComplete完成延迟任务

    override def tryComplete(): Boolean = {
    // check for each partition if it still has pending acks
    // 遍历produceMetadata中所有分区的状态
    produceMetadata.produceStatus.foreach { case (topicAndPartition, status) =>
      trace("Checking produce satisfaction for %s, current status %s"
        .format(topicAndPartition, status))
      // skip those partitions that have already been satisfied
      if (status.acksPending) {// 检测此分区是否已经满足DelayedProduce条件
      // 获取对应的分区对象
        val partitionOpt = replicaManager.getPartition(topicAndPartition.topic, topicAndPartition.partition)
        val (hasEnough, errorCode) = partitionOpt match {
          case Some(partition) =>
          // 检测这个分区的HW位置是否大于requiredOffset
            partition.checkEnoughReplicasReachOffset(status.requiredOffset)
          case None =>
            // Case A
            // 找不到这个分区的leader
            (false, Errors.UNKNOWN_TOPIC_OR_PARTITION.code)
        }
        if (errorCode != Errors.NONE.code) {// 出现异常
          // Case B.1
          status.acksPending = false
          status.responseStatus.errorCode = errorCode
        } else if (hasEnough) {// 此分区leader副本的HW大于requiredOffset
          // Case B.2
          status.acksPending = false
          status.responseStatus.errorCode = Errors.NONE.code
        }
      }
    }

    // check if each partition has satisfied at lease one of case A and case B
    // 检测全部分区是否都符合DelayedProduce的执行条件
    if (! produceMetadata.produceStatus.values.exists(p => p.acksPending))
      forceComplete()
    else
      false
  }

onComplete方法是执行DelayedProduce的真正逻辑

  override def onComplete() {
      // 根据ProduceMetadata记录的相关信息，为每个Partition产生响应状态
    val responseStatus = produceMetadata.produceStatus.mapValues(status => status.responseStatus)
    // 调用responseCallback回调函数
    responseCallback(responseStatus)
  }

最后是整个延迟任务的回调函数responseCallback

def sendResponseCallback(responseStatus: Map[TopicPartition, PartitionResponse]) {

  val mergedResponseStatus = responseStatus ++ unauthorizedRequestInfo.mapValues(_ =>
    new PartitionResponse(Errors.TOPIC_AUTHORIZATION_FAILED.code, -1, Message.NoTimestamp))
  // 标识处理ProducerRequest的过程是否出现异常
  var errorInResponse = false

  mergedResponseStatus.foreach { case (topicPartition, status) =>
    if (status.errorCode != Errors.NONE.code) {
      errorInResponse = true
      debug("Produce request with correlation id %d from client %s on partition %s failed due to %s".format(
        request.header.correlationId,
        request.header.clientId,
        topicPartition,
        Errors.forCode(status.errorCode).exceptionName))
    }
  }

  def produceResponseCallback(delayTimeMs: Int) {
      // 处理acks字段为0的情况，生产者不需要服务端返回响应
    if (produceRequest.acks == 0) {
      // no operation needed if producer request.required.acks = 0; however, if there is any error in handling
      // the request, since no response is expected by the producer, the server will close socket server so that
      // the producer client will know that some error has happened and will refresh its metadata
      if (errorInResponse) {
          // 处理ProducerRequest过程中出现异常，则向对应responseQueue中添加ReuqestChannel.CloseConnectionAction类型响应，关闭连接
        val exceptionsSummary = mergedResponseStatus.map { case (topicPartition, status) =>
          topicPartition -> Errors.forCode(status.errorCode).exceptionName
        }.mkString(", ")
        info(
          s"Closing connection due to error during produce request with correlation id ${request.header.correlationId} " +
            s"from client id ${request.header.clientId} with ack=0\n" +
            s"Topic and partition to exceptions: $exceptionsSummary"
        )
        requestChannel.closeConnection(request.processor, request)
      } else {
          // 没有出现异常，继续读取客户端的请求
        requestChannel.noOperation(request.processor, request)
      }
    } else {// 处理acks不是0的情况，即要服务端响应
    // 创建消息头
      val respHeader = new ResponseHeader(request.header.correlationId)
      val respBody = request.header.apiVersion match {// 创建消息体
        case 0 => new ProduceResponse(mergedResponseStatus.asJava)
        case version@(1 | 2) => new ProduceResponse(mergedResponseStatus.asJava, delayTimeMs, version)
        // This case shouldn't happen unless a new version of ProducerRequest is added without
        // updating this part of the code to handle it properly.
        case version => throw new IllegalArgumentException(s"Version `$version` of ProduceRequest is not handled. Code must be updated.")
      }
        // 向对应responseQueue中添加SendAction类型响应，把响应返回客户端
      requestChannel.sendResponse(new RequestChannel.Response(request, new ResponseSend(request.connectionId, respHeader, respBody)))
    }
  }

  // When this callback is triggered, the remote API call has completed
  request.apiRemoteCompleteTimeMs = SystemTime.milliseconds
}

1. 生产者发送请求向某些指定分区追加消息
2. ProducerRequest经过网络层和API层到达ReplicaManager，他会将消息交给日志存储系统进行处理，最终追加到对应的log中，同时还是检测delayedFetchPurgatory中相关key对应的DelayedFetch，满足条件则将其执行完成。
3. 日志存储系统返回追加消息的结果
4. ReplicaManager为ProducerRequest生成DelayedProduce对象，并交由delayedProducePurgatory管理
5. delayedProducePurgatory使用SystemTimer管理DelayedProduce是否超时
6. ISR集合中Follower副本发送FetchRequest请求与Leader副本同步消息，同时也会检查DelayedProduce是否符合执行条件。
7. DelayProduce执行时会调用回调函数产生ProducerResponse，并将其添加到RequestChannels中
8. 由网络层将ProduceResponse返回给客户端。