Spark Core（九）SparkContext创建过程与源码分析

1. SparkContext创建之创建TaskScheduler

   

2. 创建TaskSchedule源码

   1. 创建TaskSchedulerImpl

      // 当我们创建SparkContext的时候，这个作为主构造函数就会被执行，调用createTaskScheduler方法，创建TaskScheduler并开启它
      val (sched, ts) = SparkContext.createTaskScheduler(this, master)

   2. createTaskScheduler方法

      /**
         * 1、根据传进来的Master的URl来进行模式匹配，来选择不同的执行模式
         * 2、然后返回一个元组，元组元素为SchedulerBackend和TaskScheduler
         */
        private def createTaskScheduler(
            sc: SparkContext,
            master: String): (SchedulerBackend, TaskScheduler) = {
          import SparkMasterRegex._
      
          // When running locally, don't try to re-execute tasks on failure.
          val MAX_LOCAL_TASK_FAILURES = 1
      
          master match {
            //本地模式
            case "local" =>
              val scheduler = new TaskSchedulerImpl(sc, MAX_LOCAL_TASK_FAILURES, isLocal = true)
              val backend = new LocalBackend(sc.getConf, scheduler, 1)
              scheduler.initialize(backend)
              (backend, scheduler)
      
            case LOCAL_N_REGEX(threads) =>
              def localCpuCount: Int = Runtime.getRuntime.availableProcessors()
              // local[*] estimates the number of cores on the machine; local[N] uses exactly N threads.
              val threadCount = if (threads == "*") localCpuCount else threads.toInt
              if (threadCount <= 0) {
                throw new SparkException(s"Asked to run locally with $threadCount threads")
              }
              val scheduler = new TaskSchedulerImpl(sc, MAX_LOCAL_TASK_FAILURES, isLocal = true)
              val backend = new LocalBackend(sc.getConf, scheduler, threadCount)
              scheduler.initialize(backend)
              (backend, scheduler)
      
            case LOCAL_N_FAILURES_REGEX(threads, maxFailures) =>
              def localCpuCount: Int = Runtime.getRuntime.availableProcessors()
              // local[*, M] means the number of cores on the computer with M failures
              // local[N, M] means exactly N threads with M failures
              val threadCount = if (threads == "*") localCpuCount else threads.toInt
              val scheduler = new TaskSchedulerImpl(sc, maxFailures.toInt, isLocal = true)
              val backend = new LocalBackend(sc.getConf, scheduler, threadCount)
              scheduler.initialize(backend)
              (backend, scheduler)
            //该匹配模式就是Spark的Standalone模式，
            case SPARK_REGEX(sparkUrl) =>
              val scheduler = new TaskSchedulerImpl(sc)
              val masterUrls = sparkUrl.split(",").map("spark://" + _)
              val backend = new SparkDeploySchedulerBackend(scheduler, sc, masterUrls)
              scheduler.initialize(backend)
              (backend, scheduler)
      
            case LOCAL_CLUSTER_REGEX(numSlaves, coresPerSlave, memoryPerSlave) =>
              // Check to make sure memory requested <= memoryPerSlave. Otherwise Spark will just hang.
              val memoryPerSlaveInt = memoryPerSlave.toInt
              if (sc.executorMemory > memoryPerSlaveInt) {
                throw new SparkException(
                  "Asked to launch cluster with %d MB RAM / worker but requested %d MB/worker".format(
                    memoryPerSlaveInt, sc.executorMemory))
              }
      
              val scheduler = new TaskSchedulerImpl(sc)
              val localCluster = new LocalSparkCluster(
                numSlaves.toInt, coresPerSlave.toInt, memoryPerSlaveInt, sc.conf)
              val masterUrls = localCluster.start()
              val backend = new SparkDeploySchedulerBackend(scheduler, sc, masterUrls)
              scheduler.initialize(backend)
              backend.shutdownCallback = (backend: SparkDeploySchedulerBackend) => {
                localCluster.stop()
              }
              (backend, scheduler)
      
            case "yarn-standalone" | "yarn-cluster" =>
              if (master == "yarn-standalone") {
                logWarning(
                  "\"yarn-standalone\" is deprecated as of Spark 1.0. Use \"yarn-cluster\" instead.")
              }
              val scheduler = try {
                val clazz = Utils.classForName("org.apache.spark.scheduler.cluster.YarnClusterScheduler")
                val cons = clazz.getConstructor(classOf[SparkContext])
                cons.newInstance(sc).asInstanceOf[TaskSchedulerImpl]
              } catch {
                // TODO: Enumerate the exact reasons why it can fail
                // But irrespective of it, it means we cannot proceed !
                case e: Exception => {
                  throw new SparkException("YARN mode not available ?", e)
                }
              }
              val backend = try {
                val clazz =
                  Utils.classForName("org.apache.spark.scheduler.cluster.YarnClusterSchedulerBackend")
                val cons = clazz.getConstructor(classOf[TaskSchedulerImpl], classOf[SparkContext])
                cons.newInstance(scheduler, sc).asInstanceOf[CoarseGrainedSchedulerBackend]
              } catch {
                case e: Exception => {
                  throw new SparkException("YARN mode not available ?", e)
                }
              }
              scheduler.initialize(backend)
              (backend, scheduler)
      
            case "yarn-client" =>
              val scheduler = try {
                val clazz = Utils.classForName("org.apache.spark.scheduler.cluster.YarnScheduler")
                val cons = clazz.getConstructor(classOf[SparkContext])
                cons.newInstance(sc).asInstanceOf[TaskSchedulerImpl]
      
              } catch {
                case e: Exception => {
                  throw new SparkException("YARN mode not available ?", e)
                }
              }
      
              val backend = try {
                val clazz =
                  Utils.classForName("org.apache.spark.scheduler.cluster.YarnClientSchedulerBackend")
                val cons = clazz.getConstructor(classOf[TaskSchedulerImpl], classOf[SparkContext])
                cons.newInstance(scheduler, sc).asInstanceOf[CoarseGrainedSchedulerBackend]
              } catch {
                case e: Exception => {
                  throw new SparkException("YARN mode not available ?", e)
                }
              }
      
              scheduler.initialize(backend)
              (backend, scheduler)
      
            case MESOS_REGEX(mesosUrl) =>
              MesosNativeLibrary.load()
              val scheduler = new TaskSchedulerImpl(sc)
              val coarseGrained = sc.conf.getBoolean("spark.mesos.coarse", defaultValue = true)
              val backend = if (coarseGrained) {
                new CoarseMesosSchedulerBackend(scheduler, sc, mesosUrl, sc.env.securityManager)
              } else {
                new MesosSchedulerBackend(scheduler, sc, mesosUrl)
              }
              scheduler.initialize(backend)
              (backend, scheduler)
      
            case SIMR_REGEX(simrUrl) =>
              val scheduler = new TaskSchedulerImpl(sc)
              val backend = new SimrSchedulerBackend(scheduler, sc, simrUrl)
              scheduler.initialize(backend)
              (backend, scheduler)
      
            case zkUrl if zkUrl.startsWith("zk://") =>
              logWarning("Master URL for a multi-master Mesos cluster managed by ZooKeeper should be " +
                "in the form mesos://zk://host:port. Current Master URL will stop working in Spark 2.0.")
              createTaskScheduler(sc, "mesos://" + zkUrl)
      
            case _ =>
              throw new SparkException("Could not parse Master URL: '" + master + "'")
          }
        }
      }

   3. TaskSchedulerImpl调用start方法

       override def start() {
          super.start()
          launcherBackend.connect()
      
          // The endpoint for executors to talk to us
          val driverUrl = rpcEnv.uriOf(SparkEnv.driverActorSystemName,
            RpcAddress(sc.conf.get("spark.driver.host"), sc.conf.get("spark.driver.port").toInt),
            CoarseGrainedSchedulerBackend.ENDPOINT_NAME)
          val args = Seq(
            "--driver-url", driverUrl,
            "--executor-id", "{{EXECUTOR_ID}}",
            "--hostname", "{{HOSTNAME}}",
            "--cores", "{{CORES}}",
            "--app-id", "{{APP_ID}}",
            "--worker-url", "{{WORKER_URL}}")
          val extraJavaOpts = sc.conf.getOption("spark.executor.extraJavaOptions")
            .map(Utils.splitCommandString).getOrElse(Seq.empty)
          val classPathEntries = sc.conf.getOption("spark.executor.extraClassPath")
            .map(_.split(java.io.File.pathSeparator).toSeq).getOrElse(Nil)
          val libraryPathEntries = sc.conf.getOption("spark.executor.extraLibraryPath")
            .map(_.split(java.io.File.pathSeparator).toSeq).getOrElse(Nil)
      
          // When testing, expose the parent class path to the child. This is processed by
          // compute-classpath.{cmd,sh} and makes all needed jars available to child processes
          // when the assembly is built with the "*-provided" profiles enabled.
          val testingClassPath =
            if (sys.props.contains("spark.testing")) {
              sys.props("java.class.path").split(java.io.File.pathSeparator).toSeq
            } else {
              Nil
            }
      
          // Start executors with a few necessary configs for registering with the scheduler
          val sparkJavaOpts = Utils.sparkJavaOpts(conf, SparkConf.isExecutorStartupConf)
          val javaOpts = sparkJavaOpts ++ extraJavaOpts
          val command = Command("org.apache.spark.executor.CoarseGrainedExecutorBackend",
            args, sc.executorEnvs, classPathEntries ++ testingClassPath, libraryPathEntries, javaOpts)
          val appUIAddress = sc.ui.map(_.appUIAddress).getOrElse("")
          val coresPerExecutor = conf.getOption("spark.executor.cores").map(_.toInt)
          val appDesc = new ApplicationDescription(sc.appName, maxCores, sc.executorMemory,
            command, appUIAddress, sc.eventLogDir, sc.eventLogCodec, coresPerExecutor)
      
          //创建AppClient，
      
          client = new AppClient(sc.env.rpcEnv, masters, appDesc, this, conf)
          client.start()
          launcherBackend.setState(SparkAppHandle.State.SUBMITTED)
          waitForRegistration()
          launcherBackend.setState(SparkAppHandle.State.RUNNING)
        }

   4. AppClient调用start方法

      def start() {
          // Just launch an rpcEndpoint; it will call back into the listener.
          endpoint.set(rpcEnv.setupEndpoint("AppClient", new ClientEndpoint(rpcEnv)))
        }

   5. registerWithMaster方法

       private def registerWithMaster(nthRetry: Int) {
            registerMasterFutures.set(tryRegisterAllMasters())
            registrationRetryTimer.set(registrationRetryThread.scheduleAtFixedRate(new Runnable {
              override def run(): Unit = {
                Utils.tryOrExit {
                  if (registered.get) {
                    registerMasterFutures.get.foreach(_.cancel(true))
                    registerMasterThreadPool.shutdownNow()
                  } else if (nthRetry >= REGISTRATION_RETRIES) {
                    markDead("All masters are unresponsive! Giving up.")
                  } else {
                    registerMasterFutures.get.foreach(_.cancel(true))
                    registerWithMaster(nthRetry + 1)
                  }
                }
              }
            }, REGISTRATION_TIMEOUT_SECONDS, REGISTRATION_TIMEOUT_SECONDS, TimeUnit.SECONDS))
          }

   6. tryRegisterAllMasters方法

       /**
           *   向Master注册
           */
          private def tryRegisterAllMasters(): Array[JFuture[_]] = {
            for (masterAddress <- masterRpcAddresses) yield {
              registerMasterThreadPool.submit(new Runnable {
                override def run(): Unit = try {
                  if (registered.get) {
                    return
                  }
                  logInfo("Connecting to master " + masterAddress.toSparkURL + "...")
                  val masterRef =
                    rpcEnv.setupEndpointRef(Master.SYSTEM_NAME, masterAddress, Master.ENDPOINT_NAME)
                  masterRef.send(RegisterApplication(appDescription, self))
                } catch {
                  case ie: InterruptedException => // Cancelled
                  case NonFatal(e) => logWarning(s"Failed to connect to master $masterAddress", e)
                }
              })
            }
          }