集群部署
Standalone模式
启动Master
入口类为:org.apache.spark.deploy.master.Master
(可在SPARK_HOME/sbin/start-master.sh
找到)
首先进入main
函数
def main(argStrings: Array[String]) {
//设置线程的异常处理类
Thread.setDefaultUncaughtExceptionHandler(new SparkUncaughtExceptionHandler(
exitOnUncaughtException = false))
Utils.initDaemon(log)
val conf = new SparkConf
val args = new MasterArguments(argStrings, conf)
//开始启动Master,返回三元组,元素包含
//1.Master的RPC通信环境对象引用
//2.Web UI端口(没有用处)
//3.REST Server端口(没有用处)
val (rpcEnv, _, _) = startRpcEnvAndEndpoint(args.host, args.port, args.webUiPort, conf)
//等待Master执行完成
rpcEnv.awaitTermination()
}
由上可知,启动Master
的核心部分为startRpcEnvAndEndpoint
方法的调用
def startRpcEnvAndEndpoint(
host: String,
port: Int,
webUiPort: Int,
conf: SparkConf): (RpcEnv, Int, Option[Int]) = {
val securityMgr = new SecurityManager(conf)
//创建处理消息的环境RpcEnv(实际类型为NettyRpcEnv)
val rpcEnv = RpcEnv.create(SYSTEM_NAME, host, port, conf, securityMgr)
//创建Master端的Endpoint,该Endpoint是一个Master类的对象,继承了RpcEndpoint
//RPC通信环境注册Master的Endpoint,返回RpcEndpoint的引用,使用该引用收发消息
val masterEndpoint = rpcEnv.setupEndpoint(ENDPOINT_NAME,
new Master(rpcEnv, rpcEnv.address, webUiPort, securityMgr, conf))
//向Master端的RpcEndpoint发送请求,返回BoundPortsResponse对象
//BoundPortsResponse是一个样例类,包含了三个属性:rpcEndpointPort, webUIPort, restPort
val portsResponse = masterEndpoint.askSync[BoundPortsResponse](BoundPortsRequest)
(rpcEnv, portsResponse.webUIPort, portsResponse.restPort)
}
从上面可以看出,启动Master
主要包含创建RpcEnv
、注册Endpoint
、发送请求三个部分,首先是创建RpcEnv
,该过程如下:
def create(
name: String,
bindAddress: String,
advertiseAddress: String,
port: Int,
conf: SparkConf,
securityManager: SecurityManager,
numUsableCores: Int,
clientMode: Boolean): RpcEnv = {
//创建RpcEnvConfig实例
val config = RpcEnvConfig(conf, name, bindAddress, advertiseAddress, port, securityManager,
numUsableCores, clientMode)
//通过NettyRpcEnvFactory创建NettyRpcEnv实例
new NettyRpcEnvFactory().create(config)
}
很明显,RpcEnv
是通过NettyRpcEnvFactory
创建的,并且实际类型是NettyRpcEnv
def create(config: RpcEnvConfig): RpcEnv = {
val sparkConf = config.conf
// Use JavaSerializerInstance in multiple threads is safe. However, if we plan to support
// KryoSerializer in future, we have to use ThreadLocal to store SerializerInstance
val javaSerializerInstance =
new JavaSerializer(sparkConf).newInstance().asInstanceOf[JavaSerializerInstance]
//创建NettyRpcEnv实例
val nettyEnv =
new NettyRpcEnv(sparkConf, javaSerializerInstance, config.advertiseAddress,
config.securityManager, config.numUsableCores)
if (!config.clientMode) {
//定义NettyRpcEnv的启动函数
val startNettyRpcEnv: Int => (NettyRpcEnv, Int) = {
actualPort =>
nettyEnv.startServer(config.bindAddress, actualPort)
(nettyEnv, nettyEnv.address.port)
}
try {
//启动NettyRpcEnv即Master的通信服务,该过程主要是不断尝试用不同的端口启动服务,直到该端口可用
Utils.startServiceOnPort(config.port, startNettyRpcEnv, sparkConf, config.name)._1
} catch {
case NonFatal(e) =>
nettyEnv.shutdown()
throw e
}
}
nettyEnv
}
上面说明了RpcEnv
的创建过程,接下来是向该RpcEnv
中注册Master
端的Endpoint
,首先要创建Endpoint
,该Endpoint
实际就是一个Master
类的实例,因为Master
类继承了RpcEndpoint
,创建Endpoint
之后,然后再调用RpcEnv
的setupEndpoint
方法注册该Endpoint
def setupEndpoint(name: String, endpoint: RpcEndpoint): RpcEndpointRef = {
//向Dispatcher注册Endpoint
dispatcher.registerRpcEndpoint(name, endpoint)
}
可以看出注册Endpoint
实际上是调用Dispatcher
的registerRpcEndpoint
方法注册Endpoint
。
def registerRpcEndpoint(name: String, endpoint: RpcEndpoint): NettyRpcEndpointRef = {
val addr = RpcEndpointAddress(nettyEnv.address, name)
//创建Master的Endpoint引用
val endpointRef = new NettyRpcEndpointRef(nettyEnv.conf, addr, nettyEnv)
synchronized {
if (stopped) {
throw new IllegalStateException("RpcEnv has been stopped")
}
//将Endpoint及其引用封装成EndpointData,并且保存在存储EndpointData的Hash表中
if (endpoints.putIfAbsent(name, new EndpointData(name, endpoint, endpointRef)) != null) {
throw new IllegalArgumentException(s"There is already an RpcEndpoint called $name")
}
val data = endpoints.get(name)
//Endpoint及其引用保存在存储Endpoint引用的Hash表中
endpointRefs.put(data.endpoint, data.ref)
//将上述封装的EndpointData放入阻塞队列中
receivers.offer(data) // for the OnStart message
}
endpointRef
}
其中,封装EndpointData
时会创建一个Inbox
,实例化Inbox
时,会向里面放入OnStart
作为该Endpoint
的Inbox
的第一条消息。
inbox.synchronized {
messages.add(OnStart)
}
Dispatcher
在创建NettyRpcEnv
时初始化,初始化时会新建一个线程池来处理Inbox
中的消息
private val threadpool: ThreadPoolExecutor = {
val availableCores =
if (numUsableCores > 0) numUsableCores else Runtime.getRuntime.availableProcessors()
val numThreads = nettyEnv.conf.getInt("spark.rpc.netty.dispatcher.numThreads",
math.max(2, availableCores))
val pool = ThreadUtils.newDaemonFixedThreadPool(numThreads, "dispatcher-event-loop")
for (i <- 0 until numThreads) {
pool.execute(new MessageLoop)
}
pool
}
上面提到Inbox
的第一个消息为OnStart
,当消息为OnStart
时,消息处理线程会调用Master
实例(Endpoint
)的onStart
方法
override def onStart(): Unit = {
logInfo("Starting Spark master at " + masterUrl)
logInfo(s"Running Spark version ${org.apache.spark.SPARK_VERSION}")
webUi = new MasterWebUI(this, webUiPort)
webUi.bind()
masterWebUiUrl = "http://" + masterPublicAddress + ":" + webUi.boundPort
if (reverseProxy) {
masterWebUiUrl = conf.get("spark.ui.reverseProxyUrl", masterWebUiUrl)
webUi.addProxy()
logInfo(s"Spark Master is acting as a reverse proxy. Master, Workers and " +
s"Applications UIs are available at $masterWebUiUrl")
}
//定时检查Worker是否超时,清除超时的Worker
checkForWorkerTimeOutTask = forwardMessageThread.scheduleAtFixedRate(new Runnable {
override def run(): Unit = Utils.tryLogNonFatalError {
self.send(CheckForWorkerTimeOut)
}
}, 0, WORKER_TIMEOUT_MS, TimeUnit.MILLISECONDS)
if (restServerEnabled) {
val port = conf.getInt("spark.master.rest.port", 6066)
restServer = Some(new StandaloneRestServer(address.host, port, conf, self, masterUrl))
}
//启动REST服务
restServerBoundPort = restServer.map(_.start())
masterMetricsSystem.registerSource(masterSource)
masterMetricsSystem.start()
applicationMetricsSystem.start()
// Attach the master and app metrics servlet handler to the web ui after the metrics systems are
// started.
masterMetricsSystem.getServletHandlers.foreach(webUi.attachHandler)
applicationMetricsSystem.getServletHandlers.foreach(webUi.attachHandler)
val serializer = new JavaSerializer(conf)
//在HA环境下,选择配置的对应的选主方式
val (persistenceEngine_, leaderElectionAgent_) = RECOVERY_MODE match {
case "ZOOKEEPER" =>
logInfo("Persisting recovery state to ZooKeeper")
val zkFactory =
new ZooKeeperRecoveryModeFactory(conf, serializer)
(zkFactory.createPersistenceEngine(), zkFactory.createLeaderElectionAgent(this))
case "FILESYSTEM" =>
val fsFactory =
new FileSystemRecoveryModeFactory(conf, serializer)
(fsFactory.createPersistenceEngine(), fsFactory.createLeaderElectionAgent(this))
case "CUSTOM" =>
val clazz = Utils.classForName(conf.get("spark.deploy.recoveryMode.factory"))
val factory = clazz.getConstructor(classOf[SparkConf], classOf[Serializer])
.newInstance(conf, serializer)
.asInstanceOf[StandaloneRecoveryModeFactory]
(factory.createPersistenceEngine(), factory.createLeaderElectionAgent(this))
case _ =>
(new BlackHolePersistenceEngine(), new MonarchyLeaderAgent(this))
}
persistenceEngine = persistenceEngine_
leaderElectionAgent = leaderElectionAgent_
}
启动Worker
入口类为:org.apache.spark.deploy.worker.Worker
(可在SPARK_HOME/sbin/start-slave.sh
找到)
首先进入main
函数
def main(argStrings: Array[String]) {
//设置多线程的异常处理器
Thread.setDefaultUncaughtExceptionHandler(new SparkUncaughtExceptionHandler(
exitOnUncaughtException = false))
Utils.initDaemon(log)
val conf = new SparkConf
val args = new WorkerArguments(argStrings, conf)
//创建RpcEnv和Worker的Endpoint,并注册
val rpcEnv = startRpcEnvAndEndpoint(args.host, args.port, args.webUiPort, args.cores,
args.memory, args.masters, args.workDir, conf = conf)
// With external shuffle service enabled, if we request to launch multiple workers on one host,
// we can only successfully launch the first worker and the rest fails, because with the port
// bound, we may launch no more than one external shuffle service on each host.
// When this happens, we should give explicit reason of failure instead of fail silently. For
// more detail see SPARK-20989.
val externalShuffleServiceEnabled = conf.getBoolean("spark.shuffle.service.enabled", false)
val sparkWorkerInstances = scala.sys.env.getOrElse("SPARK_WORKER_INSTANCES", "1").toInt
require(externalShuffleServiceEnabled == false || sparkWorkerInstances <= 1,
"Starting multiple workers on one host is failed because we may launch no more than one " +
"external shuffle service on each host, please set spark.shuffle.service.enabled to " +
"false or set SPARK_WORKER_INSTANCES to 1 to resolve the conflict.")
rpcEnv.awaitTermination()
}
由上可知,Worker
的启动核心部分为对startRpcEnvAndEndpoint
方法的调用
def startRpcEnvAndEndpoint(
host: String,
port: Int,
webUiPort: Int,
cores: Int,
memory: Int,
masterUrls: Array[String],
workDir: String,
workerNumber: Option[Int] = None,
conf: SparkConf = new SparkConf): RpcEnv = {
// The LocalSparkCluster runs multiple local sparkWorkerX RPC Environments
val systemName = SYSTEM_NAME + workerNumber.map(_.toString).getOrElse("")
val securityMgr = new SecurityManager(conf)
//创建处理消息的环境RpcEnv(实际类型为NettyRpcEnv)
val rpcEnv = RpcEnv.create(systemName, host, port, conf, securityMgr)
val masterAddresses = masterUrls.map(RpcAddress.fromSparkURL(_))
//创建Worker端的Endpoint,该Endpoint是一个Worker类的对象,继承了RpcEndpoint
//RPC通信环境注册Worker的Endpoint,返回RpcEndpoint的引用,使用该引用收发消息
rpcEnv.setupEndpoint(ENDPOINT_NAME, new Worker(rpcEnv, webUiPort, cores, memory