Spark源码篇-集群部署与任务提交

集群部署

Standalone模式

启动Master

入口类为：org.apache.spark.deploy.master.Master（可在SPARK_HOME/sbin/start-master.sh找到）

首先进入main函数

def main(argStrings: Array[String]) {
   
    //设置线程的异常处理类
    Thread.setDefaultUncaughtExceptionHandler(new SparkUncaughtExceptionHandler(
      exitOnUncaughtException = false))
    Utils.initDaemon(log)
    val conf = new SparkConf
    val args = new MasterArguments(argStrings, conf)
    //开始启动Master，返回三元组，元素包含
    //1.Master的RPC通信环境对象引用
    //2.Web UI端口（没有用处）
    //3.REST Server端口（没有用处）
    val (rpcEnv, _, _) = startRpcEnvAndEndpoint(args.host, args.port, args.webUiPort, conf)
    //等待Master执行完成
    rpcEnv.awaitTermination()
  }

由上可知，启动Master的核心部分为startRpcEnvAndEndpoint方法的调用

def startRpcEnvAndEndpoint(
      host: String,
      port: Int,
      webUiPort: Int,
      conf: SparkConf): (RpcEnv, Int, Option[Int]) = {
   
    val securityMgr = new SecurityManager(conf)
    //创建处理消息的环境RpcEnv（实际类型为NettyRpcEnv）
    val rpcEnv = RpcEnv.create(SYSTEM_NAME, host, port, conf, securityMgr)
    //创建Master端的Endpoint，该Endpoint是一个Master类的对象，继承了RpcEndpoint
    //RPC通信环境注册Master的Endpoint，返回RpcEndpoint的引用，使用该引用收发消息
    val masterEndpoint = rpcEnv.setupEndpoint(ENDPOINT_NAME,
      new Master(rpcEnv, rpcEnv.address, webUiPort, securityMgr, conf))
    //向Master端的RpcEndpoint发送请求，返回BoundPortsResponse对象
    //BoundPortsResponse是一个样例类，包含了三个属性：rpcEndpointPort, webUIPort, restPort
    val portsResponse = masterEndpoint.askSync[BoundPortsResponse](BoundPortsRequest)
    (rpcEnv, portsResponse.webUIPort, portsResponse.restPort)
  }

从上面可以看出，启动Master主要包含创建RpcEnv、注册Endpoint、发送请求三个部分，首先是创建RpcEnv，该过程如下：

def create(
      name: String,
      bindAddress: String,
      advertiseAddress: String,
      port: Int,
      conf: SparkConf,
      securityManager: SecurityManager,
      numUsableCores: Int,
      clientMode: Boolean): RpcEnv = {
   
    //创建RpcEnvConfig实例
    val config = RpcEnvConfig(conf, name, bindAddress, advertiseAddress, port, securityManager,
      numUsableCores, clientMode)
    //通过NettyRpcEnvFactory创建NettyRpcEnv实例
    new NettyRpcEnvFactory().create(config)
  }

很明显，RpcEnv是通过NettyRpcEnvFactory创建的，并且实际类型是NettyRpcEnv

def create(config: RpcEnvConfig): RpcEnv = {
   
    val sparkConf = config.conf
    // Use JavaSerializerInstance in multiple threads is safe. However, if we plan to support
    // KryoSerializer in future, we have to use ThreadLocal to store SerializerInstance
    val javaSerializerInstance =
      new JavaSerializer(sparkConf).newInstance().asInstanceOf[JavaSerializerInstance]
    //创建NettyRpcEnv实例
    val nettyEnv =
      new NettyRpcEnv(sparkConf, javaSerializerInstance, config.advertiseAddress,
        config.securityManager, config.numUsableCores)
    if (!config.clientMode) {
   
      //定义NettyRpcEnv的启动函数
      val startNettyRpcEnv: Int => (NettyRpcEnv, Int) = {
    actualPort =>
        nettyEnv.startServer(config.bindAddress, actualPort)
        (nettyEnv, nettyEnv.address.port)
      }
      try {
   
        //启动NettyRpcEnv即Master的通信服务，该过程主要是不断尝试用不同的端口启动服务，直到该端口可用
        Utils.startServiceOnPort(config.port, startNettyRpcEnv, sparkConf, config.name)._1
      } catch {
   
        case NonFatal(e) =>
          nettyEnv.shutdown()
          throw e
      }
    }
    nettyEnv
  }

上面说明了RpcEnv的创建过程，接下来是向该RpcEnv中注册Master端的Endpoint，首先要创建Endpoint，该Endpoint实际就是一个Master类的实例，因为Master类继承了RpcEndpoint，创建Endpoint之后，然后再调用RpcEnv的setupEndpoint方法注册该Endpoint

def setupEndpoint(name: String, endpoint: RpcEndpoint): RpcEndpointRef = {
   
    //向Dispatcher注册Endpoint
    dispatcher.registerRpcEndpoint(name, endpoint)
}

可以看出注册Endpoint实际上是调用Dispatcher的registerRpcEndpoint方法注册Endpoint。

def registerRpcEndpoint(name: String, endpoint: RpcEndpoint): NettyRpcEndpointRef = {
   
    val addr = RpcEndpointAddress(nettyEnv.address, name)
    //创建Master的Endpoint引用
    val endpointRef = new NettyRpcEndpointRef(nettyEnv.conf, addr, nettyEnv)
    synchronized {
   
      if (stopped) {
   
        throw new IllegalStateException("RpcEnv has been stopped")
      }
      //将Endpoint及其引用封装成EndpointData，并且保存在存储EndpointData的Hash表中
      if (endpoints.putIfAbsent(name, new EndpointData(name, endpoint, endpointRef)) != null) {
   
        throw new IllegalArgumentException(s"There is already an RpcEndpoint called $name")
      }
      val data = endpoints.get(name)
      //Endpoint及其引用保存在存储Endpoint引用的Hash表中
      endpointRefs.put(data.endpoint, data.ref)
      //将上述封装的EndpointData放入阻塞队列中
      receivers.offer(data)  // for the OnStart message
    }
    endpointRef
  }

其中，封装EndpointData时会创建一个Inbox，实例化Inbox时，会向里面放入OnStart作为该Endpoint的Inbox的第一条消息。

inbox.synchronized {
   
    messages.add(OnStart)
}

Dispatcher在创建NettyRpcEnv时初始化，初始化时会新建一个线程池来处理Inbox中的消息

private val threadpool: ThreadPoolExecutor = {
   
    val availableCores =
      if (numUsableCores > 0) numUsableCores else Runtime.getRuntime.availableProcessors()
    val numThreads = nettyEnv.conf.getInt("spark.rpc.netty.dispatcher.numThreads",
      math.max(2, availableCores))
    val pool = ThreadUtils.newDaemonFixedThreadPool(numThreads, "dispatcher-event-loop")
    for (i <- 0 until numThreads) {
   
      pool.execute(new MessageLoop)
    }
    pool
  }

上面提到Inbox的第一个消息为OnStart，当消息为OnStart时，消息处理线程会调用Master实例（Endpoint）的onStart方法

override def onStart(): Unit = {
   
    logInfo("Starting Spark master at " + masterUrl)
    logInfo(s"Running Spark version ${org.apache.spark.SPARK_VERSION}")
    webUi = new MasterWebUI(this, webUiPort)
    webUi.bind()
    masterWebUiUrl = "http://" + masterPublicAddress + ":" + webUi.boundPort
    if (reverseProxy) {
   
      masterWebUiUrl = conf.get("spark.ui.reverseProxyUrl", masterWebUiUrl)
      webUi.addProxy()
      logInfo(s"Spark Master is acting as a reverse proxy. Master, Workers and " +
       s"Applications UIs are available at $masterWebUiUrl")
    }
    //定时检查Worker是否超时，清除超时的Worker
    checkForWorkerTimeOutTask = forwardMessageThread.scheduleAtFixedRate(new Runnable {
   
      override def run(): Unit = Utils.tryLogNonFatalError {
   
        self.send(CheckForWorkerTimeOut)
      }
    }, 0, WORKER_TIMEOUT_MS, TimeUnit.MILLISECONDS)

    if (restServerEnabled) {
   
      val port = conf.getInt("spark.master.rest.port", 6066)
      restServer = Some(new StandaloneRestServer(address.host, port, conf, self, masterUrl))
    }
    //启动REST服务
    restServerBoundPort = restServer.map(_.start())

    masterMetricsSystem.registerSource(masterSource)
    masterMetricsSystem.start()
    applicationMetricsSystem.start()
    // Attach the master and app metrics servlet handler to the web ui after the metrics systems are
    // started.
    masterMetricsSystem.getServletHandlers.foreach(webUi.attachHandler)
    applicationMetricsSystem.getServletHandlers.foreach(webUi.attachHandler)

    val serializer = new JavaSerializer(conf)
    //在HA环境下，选择配置的对应的选主方式
    val (persistenceEngine_, leaderElectionAgent_) = RECOVERY_MODE match {
   
      case "ZOOKEEPER" =>
        logInfo("Persisting recovery state to ZooKeeper")
        val zkFactory =
          new ZooKeeperRecoveryModeFactory(conf, serializer)
        (zkFactory.createPersistenceEngine(), zkFactory.createLeaderElectionAgent(this))
      case "FILESYSTEM" =>
        val fsFactory =
          new FileSystemRecoveryModeFactory(conf, serializer)
        (fsFactory.createPersistenceEngine(), fsFactory.createLeaderElectionAgent(this))
      case "CUSTOM" =>
        val clazz = Utils.classForName(conf.get("spark.deploy.recoveryMode.factory"))
        val factory = clazz.getConstructor(classOf[SparkConf], classOf[Serializer])
          .newInstance(conf, serializer)
          .asInstanceOf[StandaloneRecoveryModeFactory]
        (factory.createPersistenceEngine(), factory.createLeaderElectionAgent(this))
      case _ =>
        (new BlackHolePersistenceEngine(), new MonarchyLeaderAgent(this))
    }
    persistenceEngine = persistenceEngine_
    leaderElectionAgent = leaderElectionAgent_
  }

启动Worker

入口类为：org.apache.spark.deploy.worker.Worker（可在SPARK_HOME/sbin/start-slave.sh找到）

首先进入main函数

def main(argStrings: Array[String]) {
   
    //设置多线程的异常处理器
    Thread.setDefaultUncaughtExceptionHandler(new SparkUncaughtExceptionHandler(
      exitOnUncaughtException = false))
    Utils.initDaemon(log)
    val conf = new SparkConf
    val args = new WorkerArguments(argStrings, conf)
    //创建RpcEnv和Worker的Endpoint，并注册
    val rpcEnv = startRpcEnvAndEndpoint(args.host, args.port, args.webUiPort, args.cores,
      args.memory, args.masters, args.workDir, conf = conf)
    // With external shuffle service enabled, if we request to launch multiple workers on one host,
    // we can only successfully launch the first worker and the rest fails, because with the port
    // bound, we may launch no more than one external shuffle service on each host.
    // When this happens, we should give explicit reason of failure instead of fail silently. For
    // more detail see SPARK-20989.
    val externalShuffleServiceEnabled = conf.getBoolean("spark.shuffle.service.enabled", false)
    val sparkWorkerInstances = scala.sys.env.getOrElse("SPARK_WORKER_INSTANCES", "1").toInt
    require(externalShuffleServiceEnabled == false || sparkWorkerInstances <= 1,
      "Starting multiple workers on one host is failed because we may launch no more than one " +
        "external shuffle service on each host, please set spark.shuffle.service.enabled to " +
        "false or set SPARK_WORKER_INSTANCES to 1 to resolve the conflict.")
    rpcEnv.awaitTermination()
  }

由上可知，Worker的启动核心部分为对startRpcEnvAndEndpoint方法的调用

def startRpcEnvAndEndpoint(
      host: String,
      port: Int,
      webUiPort: Int,
      cores: Int,
      memory: Int,
      masterUrls: Array[String],
      workDir: String,
      workerNumber: Option[Int] = None,
      conf: SparkConf = new SparkConf): RpcEnv = {
   

    // The LocalSparkCluster runs multiple local sparkWorkerX RPC Environments
    val systemName = SYSTEM_NAME + workerNumber.map(_.toString).getOrElse("")
    val securityMgr = new SecurityManager(conf)
    //创建处理消息的环境RpcEnv（实际类型为NettyRpcEnv）
    val rpcEnv = RpcEnv.create(systemName, host, port, conf, securityMgr)
    val masterAddresses = masterUrls.map(RpcAddress.fromSparkURL(_))
    //创建Worker端的Endpoint，该Endpoint是一个Worker类的对象，继承了RpcEndpoint
    //RPC通信环境注册Worker的Endpoint，返回RpcEndpoint的引用，使用该引用收发消息
    rpcEnv.setupEndpoint(ENDPOINT_NAME, new Worker(rpcEnv, webUiPort, cores, memory