kafka 副本同步-ReplicaFetcher

KafkaServer.startup
replicaManager = new ReplicaManager
//follower partition是如何拉取leader partition的数据的？
  val replicaFetcherManager = new ReplicaFetcherManager
  ...
  replicaFetcherManager.addFetcherForPartitions
  def addFetcherForPartitions(partitionAndOffsets: Map[TopicPartition, BrokerAndInitialOffset]) {
    mapLock synchronized {

      //分组：partition+broker = key
      //按照key进行分区 然后去计算出来拉取的任务（线程）
      //如果leader partition在同一个broker上面，只需要启动一个线程就可以了。

      //如果没有进行分组的话，是什么样的一个情况呢？
      //肯定就是一个follower启动一个 fetch任务。一个任务就是一个线程  （100个follower  100线程）
      //f1 f2 f3 f4 -> leader partiton 都在 hadoop1 这样的话，一个线程就干4个分区的活
      //很大的会减少，我们的线程的数量
      val partitionsPerFetcher = partitionAndOffsets.groupBy{ case(topicAndPartition, brokerAndInitialOffset) =>
        BrokerAndFetcherId(brokerAndInitialOffset.broker, getFetcherId(topicAndPartition.topic, topicAndPartition.partition))}
      //遍历所有的抓取任务
      for ((brokerAndFetcherId, partitionAndOffsets) <- partitionsPerFetcher) {
        var fetcherThread: AbstractFetcherThread = null
        fetcherThreadMap.get(brokerAndFetcherId) match {
          case Some(f) => fetcherThread = f
          case None =>
            //为每个任务都创建一个线程
            fetcherThread = createFetcherThread(brokerAndFetcherId.fetcherId, brokerAndFetcherId.broker)
            fetcherThreadMap.put(brokerAndFetcherId, fetcherThread)
            //启动线程
            fetcherThread.start
        }

        fetcherThreadMap(brokerAndFetcherId).addPartitions(partitionAndOffsets.map { case (tp, brokerAndInitOffset) =>
          tp -> brokerAndInitOffset.initOffset
        })
      }
    }
-> AbstractFetcherThread extends ShutdownableThread.run -> doWork
  -> AbstractFetcherThread.doWrok()
    override def doWork() {

    val fetchRequest = inLock(partitionMapLock) {
      //构建请求
      val fetchRequest = buildFetchRequest(partitionStates.partitionStates.asScala.map { state =>
        state.topicPartition -> state.value
      })
      if (fetchRequest.isEmpty) {
        trace("There are no active partitions. Back off for %d ms before sending a fetch request".format(fetchBackOffMs))
        partitionMapCond.await(fetchBackOffMs, TimeUnit.MILLISECONDS)
      }
      fetchRequest
    }
    if (!fetchRequest.isEmpty)
      //TODO 对请求进行处理
      processFetchRequest(fetchRequest)
  }

processFetchRequest(){
      //调用了fetch这个方法，然后我们知道
      //这个方法里面肯定是发送了网络请求
      //给leader partition所在的服务器。
      responseData = fetch(fetchRequest)
}

follower发起fetch请求

responseData = fetch(fetchRequest) -> ReplicaFetcherThread.fetch
    //发送请求 ApiKeys.FETCH
    val clientResponse = sendRequest(ApiKeys.FETCH, Some(fetchRequestVersion), fetchRequest.underlying)
      //发送请求之前，最后把请求封装成ClientRequest对象
        val clientRequest = new ClientRequest(time.milliseconds(), true, send, null)
        //TODO 把请求给发送出去
        //Producer
        networkClient.blockingSendAndReceive(clientRequest)(time)
           // 调用NetworkClient向kafka服务端发送数据，与producer一致
           client.send(request, time.milliseconds())
           -> KafkaApi.handle.ApiKeys.FETCH
---------------------------------------------------------------------------
        follower发送过来拉取数据的请求（同步数据）
        case ApiKeys.FETCH => handleFetchRequest(request)
 ->   def handleFetchRequest(request: RequestChannel.Request) {
    //获取到请求
    val fetchRequest = request.requestObj.asInstanceOf[FetchRequest]

    val (existingAndAuthorizedForDescribeTopics, nonExistingOrUnauthorizedForDescribeTopics) = fetchRequest.requestInfo.partition {
      case (topicAndPartition, _) => authorize(request.session, Describe, new Resource(auth.Topic, topicAndPartition.topic)) && metadataCache.contains(topicAndPartition.topic)
    }

    val (authorizedRequestInfo, unauthorizedForReadRequestInfo) = existingAndAuthorizedForDescribeTopics.partition {
      case (topicAndPartition, _) => authorize(request.session, Read, new Resource(auth.Topic, topicAndPartition.topic))
    }

    val nonExistingOrUnauthorizedForDescribePartitionData = nonExistingOrUnauthorizedForDescribeTopics.map { case (tp, _) =>
      (tp, FetchResponsePartitionData(Errors.UNKNOWN_TOPIC_OR_PARTITION.code, -1, MessageSet.Empty))
    }
    val unauthorizedForReadPartitionData = unauthorizedForReadRequestInfo.map { case (tp, _) =>
      (tp, FetchResponsePartitionData(Errors.TOPIC_AUTHORIZATION_FAILED.code, -1, MessageSet.Empty))
    }

    // the callback for sending a fetch response
    def sendResponseCallback(responsePartitionData: Seq[(TopicAndPartition, FetchResponsePartitionData)]) {

      val convertedPartitionData =
        // Need to down-convert message when consumer only takes magic value 0.
        if (fetchRequest.versionId <= 1) {
          responsePartitionData.map { case (tp, data) =>

            // We only do down-conversion when:
            // 1. The message format version configured for the topic is using magic value > 0, and
            // 2. The message set contains message whose magic > 0
            // This is to reduce the message format conversion as much as possible. The conversion will only occur
            // when new message format is used for the topic and we see an old request.
            // Please note that if the message format is changed from a higher version back to lower version this
            // test might break because some messages in new message format can be delivered to consumers before 0.10.0.0
            // without format down conversion.
            val convertedData = if (replicaManager.getMessageFormatVersion(tp).exists(_ > Message.MagicValue_V0) &&
              !data.messages.isMagicValueInAllWrapperMessages(Message.MagicValue_V0)) {
              trace(s"Down converting message to V0 for fetch request from ${fetchRequest.clientId}")
              new FetchResponsePartitionData(data.error, data.hw, data.messages.asInstanceOf[FileMessageSet].toMessageFormat(Message.MagicValue_V0))
            } else data

            tp -> convertedData
          }
        } else responsePartitionData

      val mergedPartitionData = convertedPartitionData ++ unauthorizedForReadPartitionData ++ nonExistingOrUnauthorizedForDescribePartitionData

      mergedPartitionData.foreach { case (topicAndPartition, data) =>
        if (data.error != Errors.NONE.code)
          debug(s"Fetch request with correlation id ${fetchRequest.correlationId} from client ${fetchRequest.clientId} " +
            s"on partition $topicAndPartition failed due to ${Errors.forCode(data.error).exceptionName}")
        // record the bytes out metrics only when the response is being sent
        BrokerTopicStats.getBrokerTopicStats(topicAndPartition.topic).bytesOutRate.mark(data.messages.sizeInBytes)
        BrokerTopicStats.getBrokerAllTopicsStats().bytesOutRate.mark(data.messages.sizeInBytes)
      }

      // 调用回调函数，封装响应，并加入响应队列，后面与producer一致
      def fetchResponseCallback(delayTimeMs: Int) {
        trace(s"Sending fetch response to client ${fetchRequest.clientId} of " +
          s"${convertedPartitionData.map { case (_, v) => v.messages.sizeInBytes }.sum} bytes")
        //封装出来响应
        val response = FetchResponse(fetchRequest.correlationId, mergedPartitionData.toSeq, fetchRequest.versionId, delayTimeMs)
        //TODO
        requestChannel.sendResponse(new RequestChannel.Response(request, new FetchResponseSend(request.connectionId, response)))
      }

      // When this callback is triggered, the remote API call has completed
      request.apiRemoteCompleteTimeMs = SystemTime.milliseconds

      if (fetchRequest.isFromFollower) {
        //We've already evaluated against the quota and are good to go. Just need to record it now.
        val responseSize = sizeOfThrottledPartitions(fetchRequest, mergedPartitionData, quotas.leader)
        quotas.leader.record(responseSize)
        fetchResponseCallback(0)
      } else {
        val responseSize = FetchResponse.responseSize(FetchResponse.batchByTopic(mergedPartitionData),
          fetchRequest.versionId)
        quotas.fetch.recordAndMaybeThrottle(request.session.sanitizedUser, fetchRequest.clientId, responseSize, fetchResponseCallback)
      }
    }
    //非正常的请求
    if (authorizedRequestInfo.isEmpty)
      sendResponseCallback(Seq.empty)
    else {
      // call the replica manager to fetch messages from the local replica
      //拉取数据
      replicaManager.fetchMessages(
        fetchRequest.maxWait.toLong,
        fetchRequest.replicaId,
        fetchRequest.minBytes,
        fetchRequest.maxBytes,
        fetchRequest.versionId <= 2,
        authorizedRequestInfo,
        replicationQuota(fetchRequest),
        sendResponseCallback)
    }
  }