CC00079.spark——|Hadoop&Spark.V05|——|Spark.v05|Spark 原理 源码|Master Worker解析&模拟程序|

一、模拟程序

### --- 编程代码实现：

package org.apache.spark.deploy

import java.text.SimpleDateFormat
import java.util.{Date, Locale}
import java.util.concurrent.TimeUnit
import org.apache.spark.{SecurityManager, SparkConf}
import org.apache.spark.rpc.{RpcAddress, RpcEnv, ThreadSafeRpcEndpoint}
import org.apache.spark.util.{ThreadUtils, Utils}

private class SparkWorker(
                           override val rpcEnv: RpcEnv,
                           cores: Int,
                           memory: Int,
                           masterRpcAddresses: Array[RpcAddress],
                           endpointName: String,
                           val conf: SparkConf,
                           val securityMgr: SecurityManager)
  extends ThreadSafeRpcEndpoint {
  private val host = rpcEnv.address.host
  private val port = rpcEnv.address.port
  private val workerId = generateWorkerId()
  // 心跳间隔时间，15s一次心跳
  private val HEARTBEAT_MILLIS = 60 * 1000 / 4
  private val forwordMessageScheduler = ThreadUtils.newDaemonSingleThreadScheduledExecutor("fake worker-forward-messagescheduler")
  override def onStart(): Unit = {val info = "Starting Spark worker %s:%d with %d cores, %sM RAM".format(host, port, cores, memory)
    println(info)
    masterRpcAddresses.map { masterAddress =>
      val masterEndpoint = rpcEnv.setupEndpointRef(masterAddress, SparkMaster.ENDPOINT_NAME)
      masterEndpoint.send(TestAdd(100, 200))
      masterEndpoint.send(RegisterWorker(workerId, host, port, self, cores, memory, masterAddress))
    }
  }

  override def receive: PartialFunction[Any, Unit] = {
    case RegisteredWorker(master, masterAddress) => println(s"RegisteredWorker: master=$master masterAddress=$masterAddress")
      forwordMessageScheduler.scheduleAtFixedRate(new Runnable {
        override def run(): Unit = Utils.tryLogNonFatalError {
          master.send(Heartbeat(workerId, self))
        }
      }, 0, HEARTBEAT_MILLIS, TimeUnit.MILLISECONDS)
    case _ => println("Worker 接收到 Unknown Message")
  }
  private def createDateFormat = new SimpleDateFormat("yyyyMMddHHmmss", Locale.US)
  private def generateWorkerId(): String = {
    "worker-%s-%s-%d".format(createDateFormat.format(new Date), host, port)
  }
}

object SparkWorker {
  val SYSTEM_NAME = "fakeWorker"
  val ENDPOINT_NAME = "Worker"
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf
    val host = "wangsu"
    val port = 9998
    val systemName = SYSTEM_NAME
    val securityMgr = new SecurityManager(conf)
    val rpcEnv = RpcEnv.create(systemName, host, port, conf, securityMgr)
    // 真实条件下是从输入参数中获取的，这里是直接赋值的
    val masterAddresses = Array(RpcAddress("wangsu", 9999))
    rpcEnv.setupEndpoint(ENDPOINT_NAME, new SparkWorker(rpcEnv, 4, 4096, masterAddresses, ENDPOINT_NAME, conf, securityMgr))
    rpcEnv.awaitTermination()
  }
}

package org.apache.spark.deploy

import org.apache.spark.{SecurityManager, SparkConf}
import org.apache.spark.rpc.{RpcAddress, RpcEndpointRef, RpcEnv, ThreadSafeRpcEndpoint}

private class SparkMaster(
                           override val rpcEnv: RpcEnv,
                           address: RpcAddress,
                           val securityMgr: SecurityManager,
                           val conf: SparkConf)
  extends ThreadSafeRpcEndpoint {
  override def onConnected(remoteAddress: RpcAddress): Unit = {
    super.onConnected(remoteAddress)
    println(s"$remoteAddress is connected")
  }
  override def onStart(): Unit = {println("fake Master is starting ")
  }
  override def receive: PartialFunction[Any, Unit] = {
    case TestAdd(i, j) => println(s"${i + j}")
    case TestSub(i, j) => println(s"${i - j}")
    case RegisterWorker(workerId, host, port, workerRef, cores, memory, masterAddress) =>
      println(s"RegisterWorker: workId=$workerId workAddress=$host:$port cores=$cores memory=$memory")
      workerRef.send(RegisteredWorker(self, masterAddress))
    case Heartbeat(workerId, worker) => println(s"Heartbeat: 接收到${workerId}心跳")
    case _ => println("Master 接收到 Unknown Message")
  }
  override def onDisconnected(remoteAddress: RpcAddress): Unit = {println(s"$remoteAddress is Disconnected")
  }
}

object SparkMaster {
  val SYSTEM_NAME = "fakeMaster"
  val ENDPOINT_NAME = "Master"
  def main(args: Array[String]): Unit = {
    val host = "wangsu"
    val port = 9999
    val conf = new SparkConf
    val securityMgr = new SecurityManager(conf)
    val rpcEnv = RpcEnv.create(SYSTEM_NAME, host, port, conf, securityMgr)
    val masterEndpoint = rpcEnv.setupEndpoint(ENDPOINT_NAME,
      new SparkMaster(rpcEnv, rpcEnv.address, securityMgr, conf))
    masterEndpoint.send(TestAdd(1, 100))
    masterEndpoint.send(TestSub(108, 100))
    rpcEnv.awaitTermination()
  }
}
case class TestAdd(x: Int, y: Int)
case class TestSub(x: Int, y: Int)
case class RegisterWorker(workerId: String, host: String, port: Int, worker: RpcEndpointRef, cores: Int, memory: Int, masterAddress: RpcAddress)
case class RegisteredWorker(master: RpcEndpointRef, masterAddress: RpcAddress)
case class Heartbeat(workerId: String, worker: RpcEndpointRef)