spark源码跟踪(十一)ApplicationMaster中的关键线程

yarn cluster 模式提交的spark程序会使用Yarn集群中某个节点的container资源启动ApplicationMaster java进程,其启动命令是SparkSubmit进程根据用户提交spark任务命令的参数拼接而来。启动后跟踪代码调用过程。

一，Driver线程

查看半生对象object ApplicationMaster 中的main函数

object ApplicationMaster extends Logging {

  def main(args: Array[String]): Unit = {
    SignalUtils.registerLogger(log)
    val amArgs = new ApplicationMasterArguments(args)
    val sparkConf = new SparkConf()
    if (amArgs.propertiesFile != null) {
      Utils.getPropertiesFromFile(amArgs.propertiesFile).foreach { case (k, v) =>
        sparkConf.set(k, v)
      }
    }
    // Set system properties for each config entry. This covers two use cases:
    // - The default configuration stored by the SparkHadoopUtil class
    // - The user application creating a new SparkConf in cluster mode
    //
    // Both cases create a new SparkConf object which reads these configs from system properties.
    sparkConf.getAll.foreach { case (k, v) =>
      sys.props(k) = v
    }


    val yarnConf = new YarnConfiguration(SparkHadoopUtil.newConfiguration(sparkConf))
    master = new ApplicationMaster(amArgs, sparkConf, yarnConf)


    val ugi = sparkConf.get(PRINCIPAL) match {
      // We only need to log in with the keytab in cluster mode. In client mode, the driver
      // handles the user keytab.
      case Some(principal) if master.isClusterMode =>
        val originalCreds = UserGroupInformation.getCurrentUser().getCredentials()
        SparkHadoopUtil.get.loginUserFromKeytab(principal, sparkConf.get(KEYTAB).orNull)
        val newUGI = UserGroupInformation.getCurrentUser()


        if (master.appAttemptId == null || master.appAttemptId.getAttemptId > 1) {
          // Re-obtain delegation tokens if this is not a first attempt, as they might be outdated
          // as of now. Add the fresh tokens on top of the original user's credentials (overwrite).
          // Set the context class loader so that the token manager has access to jars
          // distributed by the user.
          Utils.withContextClassLoader(master.userClassLoader) {
            val credentialManager = new HadoopDelegationTokenManager(sparkConf, yarnConf, null)
            credentialManager.obtainDelegationTokens(originalCreds)
          }
        }


        // Transfer the original user's tokens to the new user, since it may contain needed tokens
        // (such as those user to connect to YARN).
        newUGI.addCredentials(originalCreds)
        newUGI


      case _ =>
        SparkHadoopUtil.get.createSparkUser()
    }


    ugi.doAs(new PrivilegedExceptionAction[Unit]() {
      override def run(): Unit = System.exit(master.run())
    })
  }

}

main函数中创建了 class ApplicationMaster 对象并调用器run()

执行runDriver()

// In cluster mode, used to tell the AM when the user's SparkContext has been initialized.
private val sparkContextPromise = Promise[SparkContext]()

private def runDriver(): Unit = {
  addAmIpFilter(None, System.getenv(ApplicationConstants.APPLICATION_WEB_PROXY_BASE_ENV))
  userClassThread = startUserApplication()

  // This a bit hacky, but we need to wait until the spark.driver.port property has
  // been set by the Thread executing the user class.
  logInfo("Waiting for spark context initialization...")
  val totalWaitTime = sparkConf.get(AM_MAX_WAIT_TIME)
  try {
    val sc = ThreadUtils.awaitResult(sparkContextPromise.future,
      Duration(totalWaitTime, TimeUnit.MILLISECONDS))
    if (sc != null) {
      val rpcEnv = sc.env.rpcEnv


      val userConf = sc.getConf
      val host = userConf.get(DRIVER_HOST_ADDRESS)
      val port = userConf.get(DRIVER_PORT)
      registerAM(host, port, userConf, sc.ui.map(_.webUrl), appAttemptId)


      val driverRef = rpcEnv.setupEndpointRef(
        RpcAddress(host, port),
        YarnSchedulerBackend.ENDPOINT_NAME)
      createAllocator(driverRef, userConf, rpcEnv, appAttemptId, distCacheConf)
    } else {
      // Sanity check; should never happen in normal operation, since sc should only be null
      // if the user app did not create a SparkContext.
      throw new IllegalStateException("User did not initialize spark context!")
    }
    resumeDriver()
    userClassThread.join()
  } catch {
    case e: SparkException if e.getCause().isInstanceOf[TimeoutException] =>
      logError(
        s"SparkContext did not initialize after waiting for $totalWaitTime ms. " +
         "Please check earlier log output for errors. Failing the application.")
      finish(FinalApplicationStatus.FAILED,
        ApplicationMaster.EXIT_SC_NOT_INITED,
        "Timed out waiting for SparkContext.")
  } finally {
    resumeDriver()
  }
}

startUserApplication()启动了大名鼎鼎的Driver线程。

/**
 * Start the user class, which contains the spark driver, in a separate Thread.
 * If the main routine exits cleanly or exits with System.exit(N) for any N
 * we assume it was successful, for all other cases we assume failure.
 *
 * Returns the user thread that was started.
 */
private def startUserApplication(): Thread = {
  logInfo("Starting the user application in a separate Thread")


  var userArgs = args.userArgs
  if (args.primaryPyFile != null && args.primaryPyFile.endsWith(".py")) {
    // When running pyspark, the app is run using PythonRunner. The second argument is the list
    // of files to add to PYTHONPATH, which Client.scala already handles, so it's empty.
    userArgs = Seq(args.primaryPyFile, "") ++ userArgs
  }
  if (args.primaryRFile != null &&
      (args.primaryRFile.endsWith(".R") || args.primaryRFile.endsWith(".r"))) {
    // TODO(davies): add R dependencies here
  }


  val mainMethod = userClassLoader.loadClass(args.userClass)
    .getMethod("main", classOf[Array[String]])


  val userThread = new Thread {
    override def run(): Unit = {
      try {
        if (!Modifier.isStatic(mainMethod.getModifiers)) {
          logError(s"Could not find static main method in object ${args.userClass}")
          finish(FinalApplicationStatus.FAILED, ApplicationMaster.EXIT_EXCEPTION_USER_CLASS)
        } else {
          mainMethod.invoke(null, userArgs.toArray)
          finish(FinalApplicationStatus.SUCCEEDED, ApplicationMaster.EXIT_SUCCESS)
          logDebug("Done running user class")
        }
      } catch {
        case e: InvocationTargetException =>
          e.getCause match {
            case _: InterruptedException =>
              // Reporter thread can interrupt to stop user class
            case SparkUserAppException(exitCode) =>
              val msg = s"User application exited with status $exitCode"
              logError(msg)
              finish(FinalApplicationStatus.FAILED, exitCode, msg)
            case cause: Throwable =>
              logError("User class threw exception: " + cause, cause)
              finish(FinalApplicationStatus.FAILED,
                ApplicationMaster.EXIT_EXCEPTION_USER_CLASS,
                "User class threw exception: " + StringUtils.stringifyException(cause))
          }
          sparkContextPromise.tryFailure(e.getCause())
      } finally {
        // Notify the thread waiting for the SparkContext, in case the application did not
        // instantiate one. This will do nothing when the user code instantiates a SparkContext
        // (with the correct master), or when the user code throws an exception (due to the
        // tryFailure above).
        sparkContextPromise.trySuccess(null)
      }
    }
  }
  userThread.setContextClassLoader(userClassLoader)
  userThread.setName("Driver")
  userThread.start()
  userThread
}

在Driver线程中，通过反射加载spark-submit命令行参数中–class 指定的主类，执行其main方法，其中定义了用户的各种rdd逻辑操作。

Driver线程启动完毕后，主线程进入阻塞状态，等待返回sc:SparkContext对象。
主线程何时被唤醒？
查找sparkContextPromis变量的使用，在ApplicationMaster伴生对象中定义sparkContextInitialized(sc: SparkContext) 函数。

该函数在SparkContext.scala 类中被调用，该语句的定义位置在类中，没有被类方法包裹，所以是在类构造函数块中。

结论：Driver线程启动后开始执行—class指定的用户主类，用户代码执行到 new SparkContext(sparkConf)语句时，Driver线程阻塞，主线程唤醒并获取到Driver线程创建的SparkContext对象。

二，Reporter线程

Driver线程阻塞，主线程唤醒后继续往下执行。

向yarn注册自身（ApplicationMaster）

 createAllocator(driverRef, userConf, rpcEnv, appAttemptId, distCacheConf)

private def createAllocator(
      driverRef: RpcEndpointRef,
      _sparkConf: SparkConf,
      rpcEnv: RpcEnv,
      appAttemptId: ApplicationAttemptId,
      distCacheConf: SparkConf): Unit = {
    // In client mode, the AM may be restarting after delegation tokens have reached their TTL. So
    // always contact the driver to get the current set of valid tokens, so that local resources can
    // be initialized below.
    if (!isClusterMode) {
      val tokens = driverRef.askSync[Array[Byte]](RetrieveDelegationTokens)
      if (tokens != null) {
        SparkHadoopUtil.get.addDelegationTokens(tokens, _sparkConf)
      }
    }

    val appId = appAttemptId.getApplicationId().toString()
    val driverUrl = RpcEndpointAddress(driverRef.address.host, driverRef.address.port,
      CoarseGrainedSchedulerBackend.ENDPOINT_NAME).toString
    val localResources = prepareLocalResources(distCacheConf)

    // Before we initialize the allocator, let's log the information about how executors will
    // be run up front, to avoid printing this out for every single executor being launched.
    // Use placeholders for information that changes such as executor IDs.
    logInfo {
      val executorMemory = _sparkConf.get(EXECUTOR_MEMORY).toInt
      val executorCores = _sparkConf.get(EXECUTOR_CORES)
      val dummyRunner = new ExecutorRunnable(None, yarnConf, _sparkConf, driverUrl, "<executorId>",
        "<hostname>", executorMemory, executorCores, appId, securityMgr, localResources,
        ResourceProfile.DEFAULT_RESOURCE_PROFILE_ID)
      dummyRunner.launchContextDebugInfo()
    }

    allocator = client.createAllocator(
      yarnConf,
      _sparkConf,
      appAttemptId,
      driverUrl,
      driverRef,
      securityMgr,
      localResources)

    // Initialize the AM endpoint *after* the allocator has been initialized. This ensures
    // that when the driver sends an initial executor request (e.g. after an AM restart),
    // the allocator is ready to service requests.
    rpcEnv.setupEndpoint("YarnAM", new AMEndpoint(rpcEnv, driverRef))

    allocator.allocateResources()
    val ms = MetricsSystem.createMetricsSystem(MetricsSystemInstances.APPLICATION_MASTER,
      sparkConf, securityMgr)
    val prefix = _sparkConf.get(YARN_METRICS_NAMESPACE).getOrElse(appId)
    ms.registerSource(new ApplicationMasterSource(prefix, allocator))
    // do not register static sources in this case as per SPARK-25277
    ms.start(false)
    metricsSystem = Some(ms)
    reporterThread = launchReporterThread()
  }

allocator.allocateResources()中涉及到很多内容，包括Container的申请/释放，Executor的启动/关闭等。
具体请参考：
链接: spark源码跟踪(十)container申请与Executor启动.

这里只关注Reporter后台线程的启动

private def launchReporterThread(): Thread = {
    val t = new Thread {
      override def run(): Unit = {
        try {
          allocationThreadImpl()
        } finally {
          allocator.stop()
        }
      }
    }
    t.setDaemon(true)
    t.setName("Reporter")
    t.start()
    logInfo(s"Started progress reporter thread with (heartbeat : $heartbeatInterval, " +
            s"initial allocation : $initialAllocationInterval) intervals")
    t
  }

Reporter线程中只干了一件事，调用 allocationThreadImpl()方法。

可以看出Reporter线程在Application正常运行期间，一直在重复调用
allocator.allocateResources()，
具体请参考：
链接: spark源码跟踪(十)container申请与Executor启动.

该循环体每200ms执行一次，所以Reporter线程实质上也承担了心跳的功能，保持ApplicationMaster与Yarn ResourceManager的连接。

最后在回到主线程中，唤醒Driver线程，并等待其执行完毕。

三，图示