上次说到任务在submit的时候会根据,资源平台的不同,反射调用不同的类来调度任务,首先说下yarn的调度入口。
找到org.apache.spark.deploy.yarn.Client
总结下:client主要是初试话参数,封装RM为创建Application所需要的信息,最后通过RPC channel 向yarn发送启动ApplicationMaster的请求
//从main方法开始
def main(argStrings: Array[String]) {
if (!sys.props.contains("SPARK_SUBMIT")) {
logWarning("WARNING: This client is deprecated and will be removed in a " +
"future version of Spark. Use ./bin/spark-submit with \"--master yarn\"")
}
// 系统环境添加yarn集群的标识
System.setProperty("SPARK_YARN_MODE", "true")
//实例化SparkConf
val sparkConf = new SparkConf
val args = new ClientArguments(argStrings, sparkConf)
// to maintain backwards-compatibility
if (!Utils.isDynamicAllocationEnabled(sparkConf)) {
sparkConf.setIfMissing("spark.executor.instances", args.numExecutors.toString)
}
//实例化Client,调用run方法
new Client(args, sparkConf).run()
}
// 向yarn的ResourceManager提交Application
def run(): Unit = {
// yarn提交Application,并放回appId
this.appId = submitApplication()
if