kafka源码解析之十二KafkaController(下篇)

12.6 KafkaController内部的listener

KafkaControler（leader）通过在zk的不同目录建立各种listener来达到对topic的管理和维护，其在zk的目录结构和对应的listener如下：

12.6.1 brokerChangeListener

/**
 * This is the zookeeper listener that triggers all the state transitions for a replica
 */
class BrokerChangeListener() extends IZkChildListener with Logging {
  this.logIdent = "[BrokerChangeListener on Controller " + controller.config.brokerId + "]: "
  def handleChildChange(parentPath : String, currentBrokerList : java.util.List[String]) {
    info("Broker change listener fired for path %s with children %s".format(parentPath, currentBrokerList.mkString(",")))
    inLock(controllerContext.controllerLock) {
      if (hasStarted.get) {
        ControllerStats.leaderElectionTimer.time {
          try {
            val curBrokerIds = currentBrokerList.map(_.toInt).toSet
            val newBrokerIds = curBrokerIds -- controllerContext.liveOrShuttingDownBrokerIds
            val newBrokerInfo = newBrokerIds.map(ZkUtils.getBrokerInfo(zkClient, _))
//筛选出newBroker
            val newBrokers = newBrokerInfo.filter(_.isDefined).map(_.get)
//筛选出deadBrokerIds
val deadBrokerIds = controllerContext.liveOrShuttingDownBrokerIds -- curBrokerIds
            controllerContext.liveBrokers = curBrokerIds.map(ZkUtils.getBrokerInfo(zkClient, _)).filter(_.isDefined).map(_.get)
            info("Newly added brokers: %s, deleted brokers: %s, all live brokers: %s"
              .format(newBrokerIds.mkString(","), deadBrokerIds.mkString(","), controllerContext.liveBrokerIds.mkString(",")))
//添加和newBroker的通信通道
            newBrokers.foreach(controllerContext.controllerChannelManager.addBroker(_))
//删除和newBroker的通信通道
            deadBrokerIds.foreach(controlleContext.controllerChannelManager.removeBroker(_))
            if(newBrokerIds.size > 0)
//尝试将该broker上的replica切换为online状态，并且恢复删除topic的流程
              controller.onBrokerStartup(newBrokerIds.toSeq)
            if(deadBrokerIds.size > 0)
//尝试将该broker上的replica切换为offline状态，并且标记该replica删除失败
              controller.onBrokerFailure(deadBrokerIds.toSeq)
          } catch {
            case e: Throwable => error("Error while handling broker changes", e)
          }
        }
      }
    }
  }
}

12.6.2 topicChangeListener

class TopicChangeListener extends IZkChildListener with Logging {
  this.logIdent = "[TopicChangeListener on Controller " + controller.config.brokerId + "]: "

  @throws(classOf[Exception])
  def handleChildChange(parentPath : String, children : java.util.List[String]) {
    inLock(controllerContext.controllerLock) {
      if (hasStarted.get) {
        try {
          val currentChildren = {
            import JavaConversions._
            debug("Topic change listener fired for path %s with children %s".format(parentPath, children.mkString(",")))
            (children: Buffer[String]).toSet
          }
//筛选出newTopics
          val newTopics = currentChildren -- controllerContext.allTopics
//筛选出deletedTopics
          val deletedTopics = controllerContext.allTopics -- currentChildren
          controllerContext.allTopics = currentChildren
//获取topic的assignment分配情况
          val addedPartitionReplicaAssignment = ZkUtils.getReplicaAssignmentForTopics(zkClient, newTopics.toSeq)
//剔除deletedTopics的replicaassignment
          controllerContext.partitionReplicaAssignment = controllerContext.partitionReplicaAssignment.filter(p =>
            !deletedTopics.contains(p._1.topic))
//增加newTopics的replicaassignment
          controllerContext.partitionReplicaAssignment.++=(addedPartitionReplicaAssignment)
          info("New topics: [%s], deleted topics: [%s], new partition replica assignment [%s]".format(newTopics,
            deletedTopics, addedPartitionReplicaAssignment))
          if(newTopics.size > 0)//创建topic
            controller.onNewTopicCreation(newTopics, addedPartitionReplicaAssignment.keySet.toSet)
        } catch {
          case e: Throwable => error("Error while handling new topic", e )
        }
      }
    }
  }

12.6.3 deleteTopicsListener

class DeleteTopicsListener() extends IZkChildListener with Logging {
  this.logIdent = "[DeleteTopicsListener on " + controller.config.brokerId + "]: "
  val zkClient = controllerContext.zkClient
  /**
   * Invoked when a topic is being deleted
   * @throws Exception On any error.
   */
  @throws(classOf[Exception])
  def handleChildChange(parentPath : String, children : java.util.List[String]) {
    inLock(controllerContext.controllerLock) {
      var topicsToBeDeleted = {
        import JavaConversions._
        (children: Buffer[String]).toSet
      }
      debug("Delete topics listener fired for topics %s to be deleted".format(topicsToBeDeleted.mkString(",")))
//过滤出不存在的topic
      val nonExistentTopics = topicsToBeDeleted.filter(t => !controllerContext.allTopics.contains(t))
      if(nonExistentTopics.size > 0) {
        warn("Ignoring request to delete non-existing topics " + nonExistentTopics.mkString(","))
        nonExistentTopics.foreach(topic => ZkUtils.deletePathRecursive(zkClient, ZkUtils.getDeleteTopicPath(topic)))
      }
//剔除不存在的topic
      topicsToBeDeleted --= nonExistentTopics
      if(topicsToBeDeleted.size > 0) {
        info("Starting topic deletion for topics " + topicsToBeDeleted.mkString(","))
        // mark topic ineligible for deletion if other state changes are in progress
        topicsToBeDeleted.foreach { topic =>
          val preferredReplicaElectionInProgress =
            controllerContext.partitionsUndergoingPreferredReplicaElection.map(_.topic).contains(topic)
          val partitionReassignmentInProgress =
            controllerContext.partitionsBeingReassigned.keySet.map(_.topic).contains(topic)
//如果topic的partition的replica正在重分配或者重新选举的话，则标识该topic不能被删除
          if(preferredReplicaElectionInProgress || partitionReassignmentInProgress)
            controller.deleteTopicManager.markTopicIneligibleForDeletion(Set(topic))
        }
//把topic交由deleteTopicManager处理
        controller.deleteTopicManager.enqueueTopicsForDeletion(topicsToBeDeleted)
      }
    }
  }
  @throws(classOf[Exception])
  def handleDataDeleted(dataPath: String) {
  }
}

12.6.4 preferredReplicaElectionListener

class PreferredReplicaElectionListener(controller: Kafk