spark 业务执行进程

spark 业务执行进程

spark的task的业务执行进程的入口类为 CoarseGrainedExecutorBackend

在CoarseGrainedExecutorBackend进程当中，同时也拉起了 WorkerWatcher线程来和当前机器的work进程进行通信

  // 这里就拉起了 CoarseGrainedExecutorBackend 线程了,这个线程对象和spark driver进行通信连接，接收执行任务
  env.rpcEnv.setupEndpoint("Executor", new CoarseGrainedExecutorBackend(
    env.rpcEnv, driverUrl, executorId, sparkHostPort, cores, userClassPath, env))
  // 这里实现了和 spark work进程的通信，当spark work进程 down掉时，这个线程监听到，然后也自动中止了
  workerUrl.foreach { url =>
    env.rpcEnv.setupEndpoint("WorkerWatcher", new WorkerWatcher(env.rpcEnv, url))
  }

通过接收相关的参数调用如下的入口，然后进行创建相关的类

private[spark] class CoarseGrainedExecutorBackend(
override val rpcEnv: RpcEnv,
driverUrl: String,
executorId: String,
hostPort: String,
cores: Int,
userClassPath: Seq[URL],
env: SparkEnv)
  extends ThreadSafeRpcEndpoint with ExecutorBackend with Logging {

  var executor: Executor = null
  @volatile var driver: Option[RpcEndpointRef] = None

  // If this CoarseGrainedExecutorBackend is changed to support multiple threads, then this may need
  // to be changed so that we don't share the serializer instance across threads
  private[this] val ser: SerializerInstance = env.closureSerializer.newInstance()

  override def onStart() {
logInfo("Connecting to driver: " + driverUrl)
// 这里开始连接到driver了
rpcEnv.asyncSetupEndpointRefByURI(driverUrl).flatMap { ref =>
  // This is a very fast action so we can use "ThreadUtils.sameThread"
  driver = Some(ref)
  // 向driver进行了注册了
  ref.ask[RegisterExecutorResponse](
RegisterExecutor(executorId, self, hostPort, cores, extractLogUrls))
}(ThreadUtils.sameThread).onComplete {
  // This is a very fast action so we can use "ThreadUtils.sameThread"
  case Success(msg) => Utils.tryLogNonFatalError {
// 注册完成后调用自身的消息
Option(self).foreach(_.send(msg)) // msg must be RegisterExecutorResponse
  }
  case Failure(e) => {
logError(s"Cannot register with driver: $driverUrl", e)
System.exit(1)
  }
}(ThreadUtils.sameThread)
  }

  def extractLogUrls: Map[String, String] = {
val prefix = "SPARK_LOG_URL_"
sys.env.filterKeys(_.startsWith(prefix))
  .map(e => (e._1.substring(prefix.length).toLowerCase, e._2))
  }
  // 这里就是消息的接收器了
  override def receive: PartialFunction[Any, Unit] = {
case RegisteredExecutor(hostname) =>
  logInfo("Successfully registered with driver")
  // 和driver注册成功了
  executor = new Executor(executorId, hostname, env, userClassPath, isLocal = false)

case RegisterExecutorFailed(message) =>
  logError("Slave registration failed: " + message)
  System.exit(1)

case LaunchTask(data) =>
  if (executor == null) {
logError("Received LaunchTask command but e