spark2.11-2.3源码10_executor源码

CoarseGrainedExecutorBackend

worker中为application启动的executor，实际上是启动了这个CoarseGrainedExecutorBackend进程

首先初始化onStart

  override def onStart() {
        logInfo("Connecting to driver: " + driverUrl)
        rpcEnv.asyncSetupEndpointRefByURI(driverUrl).flatMap { ref =>
          // This is a very fast action so we can use "ThreadUtils.sameThread"
          //获取driver的进程
          driver = Some(ref)
          //向driver发送RegisterExecutor消息
          ref.ask[Boolean](RegisterExecutor(executorId, self, hostname, cores, extractLogUrls))
        }(ThreadUtils.sameThread).onComplete {
          // This is a very fast action so we can use "ThreadUtils.sameThread"
          case Success(msg) =>
            // Always receive `true`. Just ignore it
          case Failure(e) =>
            exitExecutor(1, s"Cannot register with driver: $driverUrl", e, notifyDriver = false)
        }(ThreadUtils.sameThread)
      }

返回消息在receive中

override def receive: PartialFunction[Any, Unit] = {
//driver注册executor成功之后，会发送回来RegisteredExecutor 消息，此时，CoarseGrainedExecutorBackend会创建Executor对象，作为执行句柄，其实它的大部分功能，都是通过Executor实现的
 case RegisteredExecutor =>
      logInfo("Successfully registered with driver")
      try {
      //其实它的大部分功能，都是通过Executor实现的
        executor = new Executor(executorId, hostname, env, userClassPath, isLocal = false)
      } catch {
        case NonFatal(e) =>
          exitExecutor(1, "Unable to create executor due to " + e.getMessage, e)
      }


//TaskScheduler会向executor发送LaunchTask消息，在这里接收。
//启动task
 case LaunchTask(data) =>
      if (executor == null) {
        exitExecutor(1, "Received LaunchTask command but executor was null")
      } else {
      //处理里面数据
        val taskDesc = TaskDescription.decode(data.value)
        logInfo("Got assigned task " + taskDesc.taskId)
        用内部的执行句柄，Executor的launchTask方法启动一个task
        executor.launchTask(this, taskDesc)
      }
    }

launchTask

 def launchTask(context: ExecutorBackend, taskDescription: TaskDescription): Unit = {
 	//对于每一个task，都会创建一个TaskRunner
    val tr = new TaskRunner(context, taskDescription)
    //TaskRunner放入内存缓存
    runningTasks.put(taskDescription.taskId, tr)
    //将task封装在一个线程中，丢入线程池
    threadPool.execute(tr)
  }