Worker原理剖析与源码分析

• Master会让worker启动一个Driver Runnner线程，启动Driver线程（launchDriver）。
• Master让Worker启动一个ExecutorRunner，去启动Executor进程并向Driver反向注册（launchExecutor）。

case LaunchDriver(driverId, driverDesc) => {
      logInfo(s"Asked to launch driver $driverId")
      //创建一个Driver Runner，管理Driver 进程。
      val driver = new DriverRunner(
        conf,
        driverId,
        workDir,
        sparkHome,
        driverDesc.copy(command = Worker.maybeUpdateSSLSettings(driverDesc.command, conf)),
        self,
        workerUri,
        securityMgr)
      drivers(driverId) = driver
      driver.start()

      coresUsed += driverDesc.cores
      memoryUsed += driverDesc.mem

Driver Runner机制

private[worker] def start() = {
//这里创建了一个JAVA的线程
    new Thread("DriverRunner for " + driverId) {
      override def run() {
        try {
          //创建一个Driver目录
          val driverDir = createWorkingDirectory()
          //获取JOb需要的jar包
          val localJarFilename = downloadUserJar(driverDir)

          def substituteVariables(argument: String): String = argument match {
            case "{{WORKER_URL}}" => workerUrl
            case "{{USER_JAR}}" => localJarFilename
            case other => other
          }

          // 构建了ProessBuilder
          //启动Driver，并为其分布资源
          val builder = CommandUtils.buildProcessBuilder(driverDesc.command, securityManager,
            driverDesc.mem, sparkHome.getAbsolutePath, substituteVariables)
            //ProessBuilder启动Driver
          launchDriver(builder, driverDir, driverDesc.supervise)
        }
        catch {
          case e: Exception => finalException = Some(e)
        }

        val state =
        //对Driver的退出状态做一些处理 
          if (killed) {
            DriverState.KILLED
          } else if (finalException.isDefined) {
            DriverState.ERROR
          } else {
            finalExitCode match {
              case Some(0) => DriverState.FINISHED
              case _ => DriverState.FAILED
            }
          }

        finalState = Some(state)
        //Driver向worker发送一个DriverStateChanged事件
        worker.send(DriverStateChanged(driverId, state, finalException))
      }
    }.start()
  }