stage划分算法:必须对stage划分算法很清晰,知道自己的Application被划分了几个job,每个job被划分了几个stage,每个stage有哪些代码,只能在线上报错的信息上更快的发现问题或者性能调优。
//DAGscheduler的job调度的核心入口
private[scheduler] def handleJobSubmitted(jobId: Int,
finalRDD: RDD[_],
func: (TaskContext, Iterator[_]) => _,
partitions: Array[Int],
callSite: CallSite,
listener: JobListener,
properties: Properties) {
//使用触发job的最后一个RDD创建finalStage
var finalStage: ResultStage = null
try {
// New stage creation may throw an exception if, for example, jobs are run on a
// HadoopRDD whose underlying HDFS files have been deleted.
//将stage添加到DAGSchedule缓存中
finalStage = newResultStage(finalRDD, func, partitions, jobId, callSite)
} catch {
case e: Exception =>
logWarning("Creating new stage failed due to exception - job: " + jobId, e)
listener.jobFailed(e)
return
}
//使用finalStage创建一个Job(最后的stage就是finalStage)
val job = new ActiveJob(jobId, finalStage, callSite, listener, properties)
clearCacheLocs()
logInfo("Got job %s (%s) with %d output partitions".format(
job.jobId, callSite.shortForm, partitions.length))
logInfo("Final stage: " + finalStage + " (" + finalStage.name + ")")
logInfo("Parents of final stage: " + finalStage.parents)
logInfo("Missing parents: " + getMissingParentStages(finalStage))
val jobSubmissionTime = clock.getTimeMillis()
//将job加入到内存缓存中
jobIdToActiveJob(jobId) = job
activeJobs += job
finalStage.setActiveJob(job)
val stageIds = jobIdToStageIds(jobId).toArray
val stageInfos = stageIds.flatMap(id => stageIdToStage.get(id).map(_.latestInfo))
listenerBus.post(
SparkListenerJobStart(job.jobId, jobSubmissionTime, stageInfos, properties))
//使用submitStage提交finalStage
//这个方法的调用,其实会导致第一个stage的提交,其余的stage都储存在栈里。
submitStage(finalStage)
//提交等待的stage
submitWaitingStages()
}
//提交stage的方法
//这里其实就是stage划分算法的入口
//但是stage划分算法是submitStage()和getMissingParentStages()方法共同组成的。
private def submitStage(stage: Stage) {
val jobId = activeJobForStage(stage)
if (jobId.isDefined) {
logDebug("submitStage(" + stage + ")")
if (!waitingStages(stage) && !runningStages(stage) && !failedStages(stage)) {
//调用getMissingParentStages()获取父stage
val missing = getMissingParentStages(stage).sortBy(_.id)
logDebug("missing: " + missing)
if (missing.isEmpty) {
logInfo("Submitting " + stage + " (" + stage.rdd + "), which has no missing parents")
submitMissingTasks(stage, jobId.get)
} else {
//递归调用submitStage()方法,去提交父stage,知道最后没有父stage了。
//此时会提交stage0,其余的stage都在waitingStages里面了。
//这里的递归相当于stage算法的精髓
for (parent <- missing) {
submitStage(parent)
}
// 并且将当前stage放入waitingStages队列中
waitingStages += stage
}
}
} else {
abortStage(stage, "No active job for stage " + stage.id, None)
}
}
//获取某个Stage的父Stage
//如果发现最后一个RDD的所有依赖都是窄依赖,就不会创建新的RDD。
//但是如果这个RDD宽依赖了某个RDD,那么将会创建一个新的stage。
//并且将新的stage立即返回。
private def getMissingParentStages(stage: Stage): List[Stage] = {
val missing = new HashSet[Stage]
val visited = new HashSet[RDD[_]]
// We are manually maintaining a stack here to prevent StackOverflowError
// 定义了一个栈
val waitingForVisit = new Stack[RDD[_]]
def visit(rdd: RDD[_]) {
if (!visited(rdd)) {
visited += rdd
val rddHasUncachedPartitions = getCacheLocs(rdd).contains(Nil)
if (rddHasUncachedPartitions) {
for (dep <- rdd.dependencies) {
dep match {
//如果是宽依赖的话。
//其实对于每一个有shuffle操作的算子,底层都对应了三个RDD(MapPartitionsRDD,shuffleRDD,MapPartitionsRDD)
//shuffleRdd的map端的会划分到新的RDD
case shufDep: ShuffleDependency[_, _, _] =>
//使用宽依赖的RDD创建一个stage,并且会将isshufflemap设置为true
//默认最后一个stage不是shufflemap Stage
//但是fianalstage之前的stage都是shuffleMap stage
val mapStage = getShuffleMapStage(shufDep, stage.firstJobId)
if (!mapStage.isAvailable) {
missing += mapStage
}
//如果是窄依赖,就将RDD放入栈中
case narrowDep: NarrowDependency[_] =>
waitingForVisit.push(narrowDep.rdd)
}
}
}
}
//首先,向栈中推入了stage的最后一个RDD
waitingForVisit.push(stage.rdd)
while (waitingForVisit.nonEmpty) {
//对stage的最后一个RDD,调用Visit()方法
visit(waitingForVisit.pop())
}
missing.toList
}
//提交stage,为stage创建一批task,task数量与partition数量相同
private def submitMissingTasks(stage: Stage, jobId: Int) {
logDebug("submitMissingTasks(" + stage + ")")
// 获取partition数量
stage.pendingPartitions.clear()
val partitionsToCompute: Seq[Int] = stage.findMissingPartitions()
initialized.
if (stage.internalAccumulators.isEmpty || stage.numPartitions == partitionsToCompute.size) {
stage.resetInternalAccumulators()
}
val properties = jobIdToActiveJob(jobId).properties
//将stage加入runningStages队列
runningStages += stage
stage match {
case s: ShuffleMapStage =>
outputCommitCoordinator.stageStart(stage = s.id, maxPartitionId = s.numPartitions - 1)
case s: ResultStage =>
outputCommitCoordinator.stageStart(
stage = s.id, maxPartitionId = s.rdd.partitions.length - 1)
}
val taskIdToLocations: Map[Int, Seq[TaskLocation]] = try {
stage match {
case s: ShuffleMapStage =>
partitionsToCompute.map { id => (id, getPreferredLocs(stage.rdd, id))}.toMap
case s: ResultStage =>
val job = s.activeJob.get
partitionsToCompute.map { id =>
val p = s.partitions(id)
(id, getPreferredLocs(stage.rdd, p))
}.toMap
}
} catch {
case NonFatal(e) =>
stage.makeNewStageAttempt(partitionsToCompute.size)
listenerBus.post(SparkListenerStageSubmitted(stage.latestInfo, properties))
abortStage(stage, s"Task creation failed: $e\n${e.getStackTraceString}", Some(e))
runningStages -= stage
return
}
stage.makeNewStageAttempt(partitionsToCompute.size, taskIdToLocations.values.toSeq)
listenerBus.post(SparkListenerStageSubmitted(stage.latestInfo, properties))
var taskBinary: Broadcast[Array[Byte]] = null
try {
case stage: ShuffleMapStage =>
closureSerializer.serialize((stage.rdd, stage.shuffleDep): AnyRef).array()
case stage: ResultStage =>
closureSerializer.serialize((stage.rdd, stage.func): AnyRef).array()
}
taskBinary = sc.broadcast(taskBinaryBytes)
} catch {
case e: NotSerializableException =>
abortStage(stage, "Task not serializable: " + e.toString, Some(e))
runningStages -= stage
return
case NonFatal(e) =>
abortStage(stage, s"Task serialization failed: $e\n${e.getStackTraceString}", Some(e))
runningStages -= stage
return
}
//为stage创建指定数量的task
val tasks: Seq[Task[_]] = try {
stage match {
//除了final Stage不是shuffle Stage。
case stage: ShuffleMapStage =>
partitionsToCompute.map { id =>
//给每一个partition创建一个task
//给每个task最佳位置
val locs = taskIdToLocations(id)
val part = stage.rdd.partitions(id)
//给shuffle Stage创建ShuffleStageTask
new ShuffleMapTask(stage.id, stage.latestInfo.attemptId,
taskBinary, part, locs, stage.internalAccumulators)
}
//不是S户发放了 Stage就是finalStage。那么创建ResultStage
case stage: ResultStage =>
val job = stage.activeJob.get
partitionsToCompute.map { id =>
//给每一个partition创建一个task
//给每个task最佳位置(就是从stage的最后位置开始找,哪个RDD的Partition被Cache了,或被checkPoint了,那么task的最佳位置就是RDD被Cache或者被CheckPoint的位置)
val p: Int = stage.partitions(id)
val part = stage.rdd.partitions(p)
val locs = taskIdToLocations(id)
new ResultTask(stage.id, stage.latestInfo.attemptId,
taskBinary, part, locs, id, stage.internalAccumulators)
}
}
} catch {
case NonFatal(e) =>
abortStage(stage, s"Task creation failed: $e\n${e.getStackTraceString}", Some(e))
runningStages -= stage
return
}
if (tasks.size > 0) {
logInfo("Submitting " + tasks.size + " missing tasks from " + stage + " (" + stage.rdd + ")")
stage.pendingPartitions ++= tasks.map(_.partitionId)
logDebug("New pending partitions: " + stage.pendingPartitions)
taskScheduler.submitTasks(new TaskSet(
tasks.toArray, stage.id, stage.latestInfo.attemptId, jobId, properties))
stage.latestInfo.submissionTime = Some(clock.getTimeMillis())
} else {
markStageAsFinished(stage, None)
val debugString = stage match {
case stage: ShuffleMapStage =>
s"Stage ${stage} is actually done; " +
s"(available: ${stage.isAvailable}," +
s"available outputs: ${stage.numAvailableOutputs}," +
s"partitions: ${stage.numPartitions})"
case stage : ResultStage =>
s"Stage ${stage} is actually done; (partitions: ${stage.numPartitions})"
}
logDebug(debugString)
}
}
stage划分算法总结:
- 从finalstage倒推
- 通过宽依赖进行stage划分
- 通过递归,优先提交父stage