spark源码分析之master注册application

        master源码分析之注册application

1. case RegisterApplication源码

  //注册application,从taskschedulerImpl的start()方法发送master请求，这个方法接收
      //appclient发送注册请求到这个方法  
    case RegisterApplication(description, driver) => {
      // TODO Prevent repeated registrations from some driver
      if (state == RecoveryState.STANDBY) {
        // ignore, don't send response
        //如果状态是standby，不进行操作
      } else {
        logInfo("Registering app " + description.name)
        //创建application
        val app = createApplication(description, driver)
        //注册application
        registerApplication(app)
        logInfo("Registered app " + description.name + " with ID " + app.id)
        //持久化application
        persistenceEngine.addApplication(app)
        //反向，向sparkDeploySchedulerBackend的appClient发送消息告诉注册成功
        driver.send(RegisteredApplication(app.id, self))
        //调用任务执行器
        schedule()
      }
    }

2.调用registerApplication(app)方法

/**
    * 注册application
    * @param app
    */
  private def registerApplication(app: ApplicationInfo): Unit = {
  //获取driver地址
    val appAddress = app.driver.address
    if (addressToApp.contains(appAddress)) {
    //如果启动的appAddress已经在addressToApp中，停止运行
      logInfo("Attempted to re-register application at same address: " + appAddress)
      return
    }
    //加入spark测量系统
    applicationMetricsSystem.registerSource(app.appSource)
    //将APP加入内存缓存中
    apps += app
    //将appID加入idToApp缓存中
    idToApp(app.id) = app
    endpointToApp(app.driver) = app
    //将app加入到addressToApp中
    addressToApp(appAddress) = app
    //加入等待调度的队列，FIFO的算法
    waitingApps += app
  }

3.调用scheduler()方法，进行资源分配

/**
   * Schedule the currently available resources among waiting apps. This method will be called
   * every time a new app joins or resource availability changes.
    * 给正在等待的APP安排当前可用的资源。这个方法会在每个新的APP加入或者可用资源有变化时被调用
    *  schedule() 资源调度方法,启动driver和executor
   */
  private def schedule(): Unit = {
    if (state != RecoveryState.ALIVE) {
      //如果状态不是alive直接返回，也就是说standby master是不会进行资源的调度的
      return
    }
    // Drivers take strict precedence over executors
    //对worker状态是alive的worker进行random的shuffle方法，打乱worker的顺序，让worker启动更均衡
    val shuffledAliveWorkers = Random.shuffle(workers.toSeq.filter(_.state == WorkerState.ALIVE))
    //存活的workers的数量
    val numWorkersAlive = shuffledAliveWorkers.size
    var curPos = 0
    //driver的调度机制，遍历waitingDrivers 等待队列
    for (driver <- waitingDrivers.toList) { // iterate over a copy of waitingDrivers
      // We assign workers to each waiting driver in a round-robin fashion. For each driver, we
      // start from the last worker that was assigned a driver, and continue onwards until we have
      // explored all alive workers.
      //迭代等待转移的副本我们以轮循方式将worker分配给每个等待的driver。
      // 对于每个driver，我们从最后一名被分配driver的worker开始，并继续向前，
      // 直到我们探索了所有的活着的worker。
      var launched = false
      var numWorkersVisited = 0
      //遍历所有没被启动的driver
      while (numWorkersVisited < numWorkersAlive && !launched) {
        //shuffle 存活着的workers
        val worker = shuffledAliveWorkers(curPos)
        numWorkersVisited += 1
        if (worker.memoryFree >= driver.desc.mem && worker.coresFree >= driver.desc.cores) {
          //启动launchdriver，worker的内存，cpu core 大于 driver需要的
          launchDriver(worker, driver)
          //从内存缓存中移除已经启动的driver
          waitingDrivers -= driver
          launched = true
        }
        //将指针指向下一个worker
        curPos = (curPos + 1) % numWorkersAlive
      }
    }
    //在worker中启动executor
    startExecutorsOnWorkers()
  }