ControllerBrokerRequestBatch分析

为了提高KafkaController Leader和集群其他broker的通信效率，实现批量发送请求的功能

 

leaderAndIsrRequestMap：保存了发往指定broker的LeaderAndIsrRequest请求相关的信息

stopReplicaRequestMap： 保存了发往指定broker的StopReplicaRequest请求相关的信息

updateMetadataRequestMap：保存了发往指定broker的UpdateMetadataCacheRequest请求相关的信息

 

// 将保存相关请求的集合清空
def clear() {
  leaderAndIsrRequestMap.clear()
  stopReplicaRequestMap.clear()
  updateMetadataRequestMap.clear()
}
// 往LeaderAndIsrRequest请求的map添加相关信息
def addLeaderAndIsrRequestForBrokers(brokerIds: Seq[Int], topic: String, partition: Int,
                                     leaderIsrAndControllerEpoch: LeaderIsrAndControllerEpoch,
                                     replicas: Seq[Int], callback: AbstractRequestResponse => Unit = null) {
  val topicPartition = new TopicPartition(topic, partition)

  brokerIds.filter(_ >= 0).foreach { brokerId =>
    val result = leaderAndIsrRequestMap.getOrElseUpdate(brokerId, mutable.Map.empty)
    result.put(topicPartition, PartitionStateInfo(leaderIsrAndControllerEpoch, replicas.toSet))
  }

  addUpdateMetadataRequestForBrokers(controllerContext.liveOrShuttingDownBrokerIds.toSeq,
                                     Set(TopicAndPartition(topic, partition)))
}
// 往StopReplicaRequest请求的map添加相关信息
def addStopReplicaRequestForBrokers(brokerIds: Seq[Int], topic: String, partition: Int, deletePartition: Boolean,
                                    callback: (AbstractRequestResponse, Int) => Unit = null) {
  brokerIds.filter(b => b >= 0).foreach { brokerId =>
    stopReplicaRequestMap.getOrElseUpdate(brokerId, Seq.empty[StopReplicaRequestInfo])
    val v = stopReplicaRequestMap(brokerId)
    if(callback != null)
      stopReplicaRequestMap(brokerId) = v :+ StopReplicaRequestInfo(PartitionAndReplica(topic, partition, brokerId),
        deletePartition, (r: AbstractRequestResponse) => callback(r, brokerId))
    else
      stopReplicaRequestMap(brokerId) = v :+ StopReplicaRequestInfo(PartitionAndReplica(topic, partition, brokerId),
        deletePartition)
  }
}

// 往UpdateMetadataRequest请求的map添加相关信息
def addUpdateMetadataRequestForBrokers(brokerIds: Seq[Int],
                                       partitions: collection.Set[TopicAndPartition] = Set.empty[TopicAndPartition],
                                       callback: AbstractRequestResponse => Unit = null) {
  // 定义回调函数
  def updateMetadataRequestMapFor(partition: TopicAndPartition, beingDeleted: Boolean) {
    // 找到controllerContext中保存的该分区的leader
    val leaderIsrAndControllerEpochOpt = controllerContext.partitionLeadershipInfo.get(partition)
    leaderIsrAndControllerEpochOptmatch {
      case Some(leaderIsrAndControllerEpoch) =>
        // 获取该分区的AR集合
        val replicas = controllerContext.partitionReplicaAssignment(partition).toSet
        val partitionStateInfo = if (beingDeleted) {
          val leaderAndIsr = new LeaderAndIsr(LeaderAndIsr.LeaderDuringDelete, leaderIsrAndControllerEpoch.leaderAndIsr.isr)
          PartitionStateInfo(LeaderIsrAndControllerEpoch(leaderAndIsr, leaderIsrAndControllerEpoch.controllerEpoch), replicas)
        } else {
          PartitionStateInfo(leaderIsrAndControllerEpoch, replicas)
        }
        // 向updateMetadataCacaheRequestMap添加数据
        brokerIds.filter(b => b >= 0).foreach { brokerId =>
          updateMetadataRequestMap.getOrElseUpdate(brokerId, mutable.Map.empty[TopicPartition, PartitionStateInfo])
          updateMetadataRequestMap(brokerId).put(new TopicPartition(partition.topic, partition.partition), partitionStateInfo)
        }
      case None =>
        info("Leader not yet assigned for partition %s. Skip sendingUpdateMetadataRequest.".format(partition))
    }
  }
  // 如果指定的分区集合为空，则需要全部更新分区
  val filteredPartitions = {
    // 如果指定的分区集合为空
    val givenPartitions = if (partitions.isEmpty)
      // 返回获取所有分区的leader信息
      controllerContext.partitionLeadershipInfo.keySet
    else
      partitions // 直接返回分区
    // 过滤即将被删除的partition
    if (controller.deleteTopicManager.partitionsToBeDeleted.isEmpty)
      givenPartitions
    else
      givenPartitions-- controller.deleteTopicManager.partitionsToBeDeleted
  }
  // 将过滤后的分区信息添加到updateMetadataCacheRequestMap集合，等待发送
  if (filteredPartitions.isEmpty)
    brokerIds.filter(b => b >= 0).foreach { brokerId =>
      updateMetadataRequestMap.getOrElseUpdate(brokerId, mutable.Map.empty[TopicPartition, PartitionStateInfo])
    }
  else
    filteredPartitions.foreach(partition => updateMetadataRequestMapFor(partition, beingDeleted = false))

  controller.deleteTopicManager.partitionsToBeDeleted.foreach(partition => updateMetadataRequestMapFor(partition, beingDeleted = true))
}

 

def sendRequestsToBrokers(controllerEpoch: Int) {
  try {
    // 首先处理leaderAndIsrRequestMap
    leaderAndIsrRequestMap.foreach { case (broker, partitionStateInfos) =>
      partitionStateInfos.foreach { case (topicPartition, state) =>
        val typeOfRequest = if (broker == state.leaderIsrAndControllerEpoch.leaderAndIsr.leader) "become-leader" else "become-follower"
        stateChangeLogger.trace(("Controller %d epoch %d sending %s LeaderAndIsr request %s to broker %d " +
                                 "for partition [%s,%d]").format(controllerId, controllerEpoch, typeOfRequest,
                                                                 state.leaderIsrAndControllerEpoch, broker,
                                                                 topicPartition.topic, topicPartition.partition))
      }
      val leaderIds = partitionStateInfos.map(_._2.leaderIsrAndControllerEpoch.leaderAndIsr.leader).toSet
      val leaders = controllerContext.liveOrShuttingDownBrokers.filter(b => leaderIds.contains(b.id)).map {
        _.getNode(controller.config.interBrokerSecurityProtocol)
      }
      val partitionStates = partitionStateInfos.map { case (topicPartition, partitionStateInfo) =>
        val LeaderIsrAndControllerEpoch(leaderIsr, controllerEpoch) = partitionStateInfo.leaderIsrAndControllerEpoch
        val partitionState = new requests.PartitionState(controllerEpoch, leaderIsr.leader,
          leaderIsr.leaderEpoch, leaderIsr.isr.map(Integer.valueOf).asJava, leaderIsr.zkVersion,
          partitionStateInfo.allReplicas.map(Integer.valueOf).asJava
        )
        topicPartition -> partitionState
      }
      // 创建LeaderAndIsrRequest对象
      val leaderAndIsrRequest = new LeaderAndIsrRequest(controllerId, controllerEpoch, partitionStates.asJava, leaders.asJava)
      // 添加到队列等到被发送
      controller.sendRequest(broker, ApiKeys.LEADER_AND_ISR, None, leaderAndIsrRequest, null)
    }
    // 将集合清空
    leaderAndIsrRequestMap.clear()
    // 处理updateMetadataRequestMap信息
    updateMetadataRequestMap.foreach { case (broker, partitionStateInfos) =>

      partitionStateInfos.foreach(p => stateChangeLogger.trace(("Controller %d epoch %d sending UpdateMetadata request %s " +
        "to broker %d for partition %s").format(controllerId, controllerEpoch, p._2.leaderIsrAndControllerEpoch,
        broker, p._1)))
      val partitionStates = partitionStateInfos.map { case (topicPartition, partitionStateInfo) =>
        val LeaderIsrAndControllerEpoch(leaderIsr, controllerEpoch) = partitionStateInfo.leaderIsrAndControllerEpoch
        val partitionState = new requests.PartitionState(controllerEpoch, leaderIsr.leader,
          leaderIsr.leaderEpoch, leaderIsr.isr.map(Integer.valueOf).asJava, leaderIsr.zkVersion,
          partitionStateInfo.allReplicas.map(Integer.valueOf).asJava
        )
        topicPartition -> partitionState
      }

      val version = if (controller.config.interBrokerProtocolVersion >= KAFKA_0_10_0_IV1) 2: Short
                    else if (controller.config.interBrokerProtocolVersion >= KAFKA_0_9_0) 1: Short
                    else 0: Short
      // 创建UpdateMetadataRequest
      val updateMetadataRequest =
        if (version == 0) {
          val liveBrokers = controllerContext.liveOrShuttingDownBrokers.map(_.getNode(SecurityProtocol.PLAINTEXT))
          new UpdateMetadataRequest(controllerId, controllerEpoch, liveBrokers.asJava, partitionStates.asJava)
        }
        else {
          val liveBrokers = controllerContext.liveOrShuttingDownBrokers.map { broker =>
            val endPoints = broker.endPoints.map { case (securityProtocol, endPoint) =>
              securityProtocol -> new UpdateMetadataRequest.EndPoint(endPoint.host, endPoint.port)
            }
            new UpdateMetadataRequest.Broker(broker.id, endPoints.asJava, broker.rack.orNull)
          }
          new UpdateMetadataRequest(version, controllerId, controllerEpoch, partitionStates.asJava, liveBrokers.asJava)
        }
      // 添加到队列等到被发送
      controller.sendRequest(broker, ApiKeys.UPDATE_METADATA_KEY, Some(version), updateMetadataRequest, null)
    }
    // 将集合清空
    updateMetadataRequestMap.clear()
    // 处理stopReplicaRequestMap信息
    stopReplicaRequestMap.foreach { case (broker, replicaInfoList) =>
      val stopReplicaWithDelete = replicaInfoList.filter(_.deletePartition).map(_.replica).toSet
      val stopReplicaWithoutDelete = replicaInfoList.filterNot(_.deletePartition).map(_.replica).toSet
      debug("The stop replica request (delete = true) sent to broker %d is %s"
        .format(broker, stopReplicaWithDelete.mkString(",")))
      debug("The stop replica request (delete = false) sent to broker %d is %s"
        .format(broker, stopReplicaWithoutDelete.mkString(",")))
      replicaInfoList.foreach { r =>
        // 创建StopReplicaRequest对象
        val stopReplicaRequest = new StopReplicaRequest(controllerId, controllerEpoch, r.deletePartition,
          Set(new TopicPartition(r.replica.topic, r.replica.partition)).asJava)
        // 添加到队列等到被发送
        controller.sendRequest(broker, ApiKeys.STOP_REPLICA, None, stopReplicaRequest, r.callback)
      }
    }
    // 将集合清空
    stopReplicaRequestMap.clear()
  } catch {
    case e : Throwable => {
      if (leaderAndIsrRequestMap.nonEmpty) {
        error("Haven't been able to send leader and isr requests, current state of " +
            s"the map is $leaderAndIsrRequestMap. Exception message: $e")
      }
      if (updateMetadataRequestMap.nonEmpty) {
        error("Haven't been able to send metadata update requests, current state of " +
            s"the map is $updateMetadataRequestMap. Exception message: $e")
      }
      if (stopReplicaRequestMap.nonEmpty) {
        error("Haven't been able to send stop replica requests, current state of " +
            s"the map is $stopReplicaRequestMap. Exception message: $e")
      }
      throw new IllegalStateException(e)
    }
  }
}