Stage的创建

1.简介：任务提交调用：

private def doOnReceive(event: DAGSchedulerEvent): Unit = event match {
    case JobSubmitted(jobId, rdd, func, partitions, callSite, listener, properties) =>
      dagScheduler.handleJobSubmitted(jobId, rdd, func, partitions, callSite, listener, properties)

handleJobSubmitted方法。

 

2.createResultStage：

 private[scheduler] def handleJobSubmitted(jobId: Int,
      finalRDD: RDD[_],
      func: (TaskContext, Iterator[_]) => _,
      partitions: Array[Int],
      callSite: CallSite,
      listener: JobListener,
      properties: Properties) {
    var finalStage: ResultStage = null
    try {
      //例如，如果在已删除其基础HDFS文件的HadoopRDD上运行作业，则新阶段创建可能会引发异常
      finalStage = createResultStage(finalRDD, func, partitions, jobId, callSite)
    }

点进createResultStage：

  private def createResultStage(
      rdd: RDD[_],
      func: (TaskContext, Iterator[_]) => _,
      partitions: Array[Int],
      jobId: Int,
      callSite: CallSite): ResultStage = {
    checkBarrierStageWithDynamicAllocation(rdd)
    checkBarrierStageWithNumSlots(rdd)
    checkBarrierStageWithRDDChainPattern(rdd, partitions.toSet.size)
    //parents将会指向下一个stage序列
    val parents = getOrCreateParentStages(rdd, jobId)
    val id = nextStageId.getAndIncrement()
    val stage = new ResultStage(id, rdd, func, partitions, parents, jobId, callSite)
    stageIdToStage(id) = stage
    updateJobIdStageIdMaps(jobId, stage)
    stage
  }

点进getOrCreateParentStages：

  private def getOrCreateParentStages(rdd: RDD[_], firstJobId: Int): List[Stage] = {
    getShuffleDependencies(rdd).map { shuffleDep =>
      getOrCreateShuffleMapStage(shuffleDep, firstJobId)
    }.toList
  }

点进getShuffleDependencies：

    //从后往前寻找ShuffleDependencies，找不到就直到最开始的hadoopRDD停止
    //因为hadoopRDD的getDependencies是Nil,waitingForVisit就是空了
    private[scheduler] def getShuffleDependencies(
      rdd: RDD[_]): HashSet[ShuffleDependency[_, _, _]] = {
    val parents = new HashSet[ShuffleDependency[_, _, _]]
    val visited = new HashSet[RDD[_]]
    val waitingForVisit = new ListBuffer[RDD[_]]
    waitingForVisit += rdd
    //判断是否为空
    while (waitingForVisit.nonEmpty) {
      //弹出第1个元素，有点像pop
      val toVisit = waitingForVisit.remove(0)
      if (!visited(toVisit)) {
        visited += toVisit
        //dependencies返回rdd内的Dependencies，
        toVisit.dependencies.foreach {
          //模式匹配
          case shuffleDep: ShuffleDependency[_, _, _] =>
            //如果是，结束循环
            parents += shuffleDep
          case dependency => 
            //通配模式，向ListBuffer中加入序列，继续循环
            waitingForVisit.prepend(dependency.rdd)
        }
      }
    }
    //返回HashSet
    parents
  }

获得parents后用map对获取的ShuffleDependency使用getOrCreateShuffleMapStage方法，如果有的话：

  private def getOrCreateShuffleMapStage(
      shuffleDep: ShuffleDependency[_, _, _],
      firstJobId: Int): ShuffleMapStage = {
    //如果stage已经put进shuffleIdToMapStage，
    //可能是考虑下面看起来啥也没干的foreach方法中create了Stage然后put进shuffleIdToMapStage吧
    shuffleIdToMapStage.get(shuffleDep.shuffleId) match {
      case Some(stage) =>
        stage

      case None =>
        //为所有缺少的祖先shuffle依赖项创建阶段
        getMissingAncestorShuffleDependencies(shuffleDep.rdd).foreach { dep =>
          //尽管getMissingAncestorShuffleDependencies仅返回shuffleIdToMapStage中尚未存在的shuffle依赖项,
          //但是当我们到达foreach循环中的特定依赖项时,
          //它可能已经通过阶段创建过程添加到shuffleIdToMapStage中以用于早期依赖项.
          // 有关更多信息，请参见SPARK-13902.
          if (!shuffleIdToMapStage.contains(dep.shuffleId)) {
            createShuffleMapStage(dep, firstJobId)
          }
        }

        //最后，为给定的shuffle依赖创建一个阶段
        //这才是最后返回的，上面的foreach方法并不返回任何东西，可能是设置验证一些参数吧，不影响逻辑结果
        createShuffleMapStage(shuffleDep, firstJobId)
    }
  }

接着点进createShuffleMapStage：

 def createShuffleMapStage[K, V, C](
      shuffleDep: ShuffleDependency[K, V, C], jobId: Int): ShuffleMapStage = {
    val rdd = shuffleDep.rdd
    checkBarrierStageWithDynamicAllocation(rdd)
    checkBarrierStageWithNumSlots(rdd)
    checkBarrierStageWithRDDChainPattern(rdd, rdd.getNumPartitions)
    val numTasks = rdd.partitions.length
    //又是一个新的开始，递归创建parents
    val parents = getOrCreateParentStages(rdd, jobId)
    val id = nextStageId.getAndIncrement()
    //创建Stage
    val stage = new ShuffleMapStage(
      id, rdd, numTasks, parents, jobId, rdd.creationSite, shuffleDep, mapOutputTracker)

    stageIdToStage(id) = stage
    shuffleIdToMapStage(shuffleDep.shuffleId) = stage
    updateJobIdStageIdMaps(jobId, stage)

    if (!mapOutputTracker.containsShuffle(shuffleDep.shuffleId)) {
      //有点难看：需要在这里注册RDD与缓存和映射输出跟踪器，因为我们无法在RDD构造函数中执行它，因为分区的＃是未知的
      logInfo("Registering RDD " + rdd.id + " (" + rdd.getCreationSite + ")")
      mapOutputTracker.registerShuffle(shuffleDep.shuffleId, rdd.partitions.length)
    }
    //返回Stage
    stage
  }

点进ShuffleMapStage：

private[spark] class ShuffleMapStage(
    id: Int,
    rdd: RDD[_],
    numTasks: Int,
    parents: List[Stage],
    firstJobId: Int,
    callSite: CallSite,
    val shuffleDep: ShuffleDependency[_, _, _],
    mapOutputTrackerMaster: MapOutputTrackerMaster)
  extends Stage(id, rdd, numTasks, parents, firstJobId, callSite) 

点进Stage：

private var _latestInfo: StageInfo = StageInfo.fromStage(this, nextAttemptId)

点进formStage：

  def fromStage(
      stage: Stage,
      attemptId: Int,
      numTasks: Option[Int] = None,
      taskMetrics: TaskMetrics = null,
      taskLocalityPreferences: Seq[Seq[TaskLocation]] = Seq.empty
    ): StageInfo = {
    //获取依赖中的rdd信息
    val ancestorRddInfos = stage.rdd.getNarrowAncestors.map(RDDInfo.fromRdd)
    //将this.rdd加上
    val rddInfos = Seq(RDDInfo.fromRdd(stage.rdd)) ++ ancestorRddInfos
    val shuffleDepId = stage match {
      case sms: ShuffleMapStage => Option(sms.shuffleDep).map(_.shuffleId)
      case _ => None
    }
    new StageInfo(
      stage.id,
      attemptId,
      stage.name,
      numTasks.getOrElse(stage.numTasks),
      rddInfos,
      stage.parents.map(_.id),
      stage.details,
      taskMetrics,
      taskLocalityPreferences,
      shuffleDepId)
  }

点进getNarrowAncestors：

 private[spark] def getNarrowAncestors: Seq[RDD[_]] = {
    val ancestors = new mutable.HashSet[RDD[_]]

    def visit(rdd: RDD[_]): Unit = {
      //如果是NarrowDependency
      val narrowDependencies = rdd.dependencies.filter(_.isInstanceOf[NarrowDependency[_]])
      //注意，这里转换成依赖中的rdd，不要误看错，以为在NarrowDependency和ShuffleDependency交界处的rdd没加入
      val narrowParents = narrowDependencies.map(_.rdd)
      val narrowParentsNotVisited = narrowParents.filterNot(ancestors.contains)
      narrowParentsNotVisited.foreach { parent =>
        ancestors.add(parent)
        //递归，当rdd中的dependencies不是NarrowDependency时停止
        visit(parent)
      }
    }

    visit(this)

    //如果存在循环，请不要包含根本身
    ancestors.filterNot(_ == this).toSeq
  }

到此为止Stage创建基本结束。

 

3.总结：ShuffleDependency越多，Stage越多，任务越复杂。NarrowDependency中的rdd任务被保存在StageInfo中，可以把NarrowDependency任务放在一块儿来减少Stage的数量。