一、模拟程序
### --- 编程代码实现:
package org.apache.spark.deploy
import java.text.SimpleDateFormat
import java.util.{Date, Locale}
import java.util.concurrent.TimeUnit
import org.apache.spark.{SecurityManager, SparkConf}
import org.apache.spark.rpc.{RpcAddress, RpcEnv, ThreadSafeRpcEndpoint}
import org.apache.spark.util.{ThreadUtils, Utils}
private class SparkWorker(
override val rpcEnv: RpcEnv,
cores: Int,
memory: Int,
masterRpcAddresses: Array[RpcAddress],
endpointName: String,
val conf: SparkConf,
val securityMgr: SecurityManager)
extends ThreadSafeRpcEndpoint {
private val host = rpcEnv.address.host
private val port = rpcEnv.address.port
private val workerId = generateWorkerId()
// 心跳间隔时间,15s一次心跳
private val HEARTBEAT_MILLIS = 60 * 1000 / 4
private val forwordMessageScheduler = ThreadUtils.newDaemonSingleThreadScheduledExecutor("fake worker-forward-messagescheduler")
override def onStart(): Unit = {val info = "Starting Spark worker %s:%d with %d cores, %sM RAM".format(host, port, cores, memory)
println(info)
masterRpcAddresses.map { masterAddress =>
val masterEndpoint = rpcEnv.setupEndpointRef(masterAddress, SparkMaster.ENDPOINT_NAME)
masterEndpoint.send(TestAdd(100, 200))
masterEndpoint.send(RegisterWorker(workerId, host, port, self, cores, memory, masterAddress))
}
}
override def receive: PartialFunction[Any, Unit] = {
case RegisteredWorker(master, masterAddress) => println(s"RegisteredWorker: master=$master masterAddress=$masterAddress")
forwordMessageScheduler.scheduleAtFixedRate(new Runnable {
override def run(): Unit = Utils.tryLogNonFatalError {
master.send(Heartbeat(workerId, self))
}
}, 0, HEARTBEAT_MILLIS, TimeUnit.MILLISECONDS)
case _ => println("Worker 接收到 Unknown Message")
}
private def createDateFormat = new SimpleDateFormat("yyyyMMddHHmmss", Locale.US)
private def generateWorkerId(): String = {
"worker-%s-%s-%d".format(createDateFormat.format(new Date), host, port)
}
}
object SparkWorker {
val SYSTEM_NAME = "fakeWorker"
val ENDPOINT_NAME = "Worker"
def main(args: Array[String]): Unit = {
val conf = new SparkConf
val host = "wangsu"
val port = 9998
val systemName = SYSTEM_NAME
val securityMgr = new SecurityManager(conf)
val rpcEnv = RpcEnv.create(systemName, host, port, conf, securityMgr)
// 真实条件下是从输入参数中获取的,这里是直接赋值的
val masterAddresses = Array(RpcAddress("wangsu", 9999))
rpcEnv.setupEndpoint(ENDPOINT_NAME, new SparkWorker(rpcEnv, 4, 4096, masterAddresses, ENDPOINT_NAME, conf, securityMgr))
rpcEnv.awaitTermination()
}
}
package org.apache.spark.deploy
import org.apache.spark.{SecurityManager, SparkConf}
import org.apache.spark.rpc.{RpcAddress, RpcEndpointRef, RpcEnv, ThreadSafeRpcEndpoint}
private class SparkMaster(
override val rpcEnv: RpcEnv,
address: RpcAddress,
val securityMgr: SecurityManager,
val conf: SparkConf)
extends ThreadSafeRpcEndpoint {
override def onConnected(remoteAddress: RpcAddress): Unit = {
super.onConnected(remoteAddress)
println(s"$remoteAddress is connected")
}
override def onStart(): Unit = {println("fake Master is starting ")
}
override def receive: PartialFunction[Any, Unit] = {
case TestAdd(i, j) => println(s"${i + j}")
case TestSub(i, j) => println(s"${i - j}")
case RegisterWorker(workerId, host, port, workerRef, cores, memory, masterAddress) =>
println(s"RegisterWorker: workId=$workerId workAddress=$host:$port cores=$cores memory=$memory")
workerRef.send(RegisteredWorker(self, masterAddress))
case Heartbeat(workerId, worker) => println(s"Heartbeat: 接收到${workerId}心跳")
case _ => println("Master 接收到 Unknown Message")
}
override def onDisconnected(remoteAddress: RpcAddress): Unit = {println(s"$remoteAddress is Disconnected")
}
}
object SparkMaster {
val SYSTEM_NAME = "fakeMaster"
val ENDPOINT_NAME = "Master"
def main(args: Array[String]): Unit = {
val host = "wangsu"
val port = 9999
val conf = new SparkConf
val securityMgr = new SecurityManager(conf)
val rpcEnv = RpcEnv.create(SYSTEM_NAME, host, port, conf, securityMgr)
val masterEndpoint = rpcEnv.setupEndpoint(ENDPOINT_NAME,
new SparkMaster(rpcEnv, rpcEnv.address, securityMgr, conf))
masterEndpoint.send(TestAdd(1, 100))
masterEndpoint.send(TestSub(108, 100))
rpcEnv.awaitTermination()
}
}
case class TestAdd(x: Int, y: Int)
case class TestSub(x: Int, y: Int)
case class RegisterWorker(workerId: String, host: String, port: Int, worker: RpcEndpointRef, cores: Int, memory: Int, masterAddress: RpcAddress)
case class RegisteredWorker(master: RpcEndpointRef, masterAddress: RpcAddress)
case class Heartbeat(workerId: String, worker: RpcEndpointRef)