Executor原理示意图
Executor进程的启动
worker中为application启动的executor,实际上是启动的这个CoarseGrainedExecutorBackend进程.
源码分析:
第一步:CoarseGrainedExecutorBackend源码
源码地址:org.apache.spark.executor.CoarseGrainedExecutorBackend.scala
/**
* work中为application启动的executor,实际上是启动了CoarseGrainedExecutorBackend进程
*/
private[spark] class CoarseGrainedExecutorBackend(
override val rpcEnv: RpcEnv,
driverUrl: String,
executorId: String,
hostname: String,
cores: Int,
userClassPath: Seq[URL],
env: SparkEnv)
extends ThreadSafeRpcEndpoint with ExecutorBackend with Logging {
private[this] val stopping = new AtomicBoolean(false)
var executor: Executor = null
@volatile var driver: Option[RpcEndpointRef] = None
// If this CoarseGrainedExecutorBackend is changed to support multiple threads, then this may need
// to be changed so that we don't share the serializer instance across threads
private[this] val ser: SerializerInstance = env.closureSerializer.newInstance()
/**
* 初始化方法
* 相当于是向driver 发送RegisterExecutor
*/
override def onStart() {
logInfo("Connecting to driver: " + driverUrl)
rpcEnv.asyncSetupEndpointRefByURI(driverUrl).flatMap { ref =>
// This is a very fast action so we can use "ThreadUtils.sameThread"
//获取driver的actor
driver = Some(ref)
//向driver发送RegisterExecutor信息
ref.ask[Boolean](RegisterExecutor(executorId, self, hostname, cores, extractLogUrls))
}(ThreadUtils.sameThread).onComplete {
// This is a very fast action so we can use "ThreadUtils.sameThread"
case Success(msg) =>
// Always receive `true`. Just ignore it
case Failure(e) =>
exitExecutor(1, s"Cannot register with driver: $driverUrl", e, notifyDriver = false)
}(ThreadUtils.sameThread)
}
def extractLogUrls: Map[String, String] = {
val prefix = "SPARK_LOG_URL_"
sys.env.filterKeys(_.startsWith(prefix))
.map(e => (e._1.substring(prefix.length).toLowerCase(Locale.ROOT), e._2))
}
override def receive: PartialFunction[Any, Unit] = {
/**
* 当driver注册好executor之后 ,返回RegisteredExecutor消息
* 此时CoarseGrainedExecutorBackend会创建Executor执行句柄,大部分的功能都是通过Executor实现的
*/
case RegisteredExecutor =>
logInfo("Successfully registered with driver")
try {
executor = new Executor(executorId, hostname, env, userClassPath, isLocal = false)
} catch {
case NonFatal(e) =>
exitExecutor(1, "Unable to create executor due to " + e.getMessage, e)
}
......
}
启动task
源码分析
第一步:使用executor句柄的launchTask()方法,启动task
// 启动task
case LaunchTask(data) =>
if (executor == null) {
exitExecutor(1, "Received LaunchTask command but executor was null")
} else {
//反序列化
val taskDesc = TaskDescription.decode(data.value)
logInfo("Got assigned task " + taskDesc.taskId)
//使用executor句柄的launchTask()方法,启动task
executor.launchTask(this, taskDesc)
}
第二部:launchTask()方法
def launchTask(context: ExecutorBackend, taskDescription: TaskDescription): Unit = {
// 对于每一个task都需要创建一个taskRunner 【线程】
// TaskRunner实际上是继承Java的Runnable接口
val tr = new TaskRunner(context, taskDescription)
// 将TaskRunner放入内存缓存中,runningTasks维护运行任务列表。
runningTasks.put(taskDescription.taskId, tr)
//将task封装在一个线程中(TaskRunner),将线程丢入线程池中,然后执行
// 线程池是实现排队机制的,如果线程池内的线程暂时没有空闲,放入的线程就会排队
threadPool.execute(tr)
}