spark-core_16:初始化Driver的SparkEnv

1，通过SparkContext初始化时调用createSparkEnv()==>然后调用SparkEnv.createDriverEnv()来初始化SparkEnv对象

/**
 * Create a SparkEnv for the driver.
  * 该方法被SparkContext.createSparkEnv()方法进行调用
  * numCores:如果master是local模式会将driver对应节点cpu的线程数取出来，如果是集群模式则返回0
 */
private[spark] def createDriverEnv(
    conf: SparkConf,
    isLocal: Boolean,
    listenerBus: LiveListenerBus,
    numCores: Int,
    mockOutputCommitCoordinator: Option[OutputCommitCoordinator] = None): SparkEnv = {
  //这两个属性在sparkContext初始化时设置进去的
  assert(conf.contains("spark.driver.host"), "spark.driver.host is not set on the driver!")
  assert(conf.contains("spark.driver.port"), "spark.driver.port is not set on the driver!")
  val hostname = conf.get("spark.driver.host") //dirver的ip或hostname
  //spark.driver.port不设置默认就0，在create()创建RpcEnv时，

  会设置conf.set("spark.driver.port", rpcEnv.address.port.toString)
  val port = conf.get("spark.driver.port").toInt
  //SparkContext.DRIVER_IDENTIFIER 对应Executor的标识：值是"driver"串
  create(
    conf,
    SparkContext.DRIVER_IDENTIFIER,
    hostname,
    port,
    isDriver = true,
    isLocal = isLocal,
    numUsableCores = numCores,//集群模式则返回0
    listenerBus = listenerBus,
    mockOutputCommitCoordinator = mockOutputCommitCoordinator
  )
}

2,初始化生成SparkEnv

/**
 * Helper method to create a SparkEnv fora driver or an executor.
  * 该方法创建driver或executor的SparkEnv
  *
  * 一参数说明：
  * executorId：如是driver值是"driver"串
  * hostname:如driver:就是driver的ip或主机名
  * numUsableCores:如果master是local模式会将driver对应节点cpu的线程数取出来，如果是集群模式则返回0
  * port 对应spark.driver.port不设置默认就0,在创建SparkEnv，会给它设置RpcEnv.address.port
  *
  * 二、该方法会在createDriverEnv和createExecutorEnv方法中被调用，帮助driver和executor生成sparkEnv
  * a,createDriverEnv方法被SparkContext.createSparkEnv()方法进行调用
  * b,createExecutorEnv:方法会CoarseGrainedExecutorBackend或MesosExecutorBackend调用
  *
  * 三、create方法：创建出BlockManager会将下面对象放在构造方法中
  * a，BlockManagerMaster(BlockManagerMasterEndpoint,SparkConf,isDriver)
  *b,NettyBlockTransferService(SparkConf, securityManager, numUsableCores)
  *
  * SparkEnv的构造步骤如下：
  *    1.创建安全管理器SecurityManager
  *    2.创建给予AKKa的分布式消息系统ActorSystem;--未来版本会被移除
  *    3.创建Map任务输出跟踪器mapOutputTracker;
  *    4.实例化ShuffleManager;
  *    5.创建块传输服务BlockTransferService;
  *    6.创建BlockManagerMaster;
  *    7.创建块管理器BlockManager;
  *    8.创建广播管理器BroadcastManager;
  *    9.创建缓存管理器CacheManager;
  *    10.创建HTTP文件服务器HttpFileServer;
  *    11.创建测量系统MetricsSystem;
  *    12.创建输出提交控制器OutputCommitCoordinator;
  *    13.创建SparkEnv;
 */
private def create(
    conf: SparkConf,
    executorId: String,
    hostname: String,
    port: Int,
    isDriver: Boolean,
    isLocal: Boolean,
    numUsableCores: Int,
    listenerBus: LiveListenerBus = null,
    mockOutputCommitCoordinator:Option[OutputCommitCoordinator] = None): SparkEnv = {

  // Listener bus is only used on the driver
  if (isDriver){
    assert(listenerBus != null, "Attemptedto create driver SparkEnv with null listener bus!")
  }
  //安全管理器用于权限验证的
  val securityManager= new SecurityManager(conf)

  // Create the ActorSystem for Akka and get the port itbinds to.
  // 创建ActorSystem及返回对应actorSystem的port
  // driverActorSystemName = "sparkDriver" , executorActorSystemName= "sparkExecutor"
  val actorSystemName= if (isDriver) driverActorSystemName else executorActorSystemName
  //创建RpcEnv，在1.6开始已使用NettyRpcEnv,并且也不在使用ActorSystem
  //如果port是0话，会给RpcEnv.address.prot动态分配一个非0的端口

（查看：spark-core_17:自定义RpcEnv模拟Master和Worker通信及RpcEnv、RpcEndpoint、RpcEndpointRef源码分析）
  val rpcEnv= RpcEnv.create(actorSystemName, hostname, port, conf, securityManager,
    clientMode = !isDriver)
  val actorSystem:ActorSystem =
    if (rpcEnv.isInstanceOf[AkkaRpcEnv]){
      rpcEnv.asInstanceOf[AkkaRpcEnv].actorSystem
    } else {
      val actorSystemPort=
        if (port== 0 || rpcEnv.address == null) {
          port
        } else {
          rpcEnv.address.port + 1
        }
      // Create a ActorSystem for legacy codes
      //该方法返回tuple(ActorSystem,ActorSystem的port),同时将ActorSystem引用给当前变量actorSystem

(查看：spark-core_18:Akka分布式通信案例、spark-core_19:ActorSystem的初始化源码分析)
      AkkaUtils.createActorSystem(
        actorSystemName + "ActorSystem",
        hostname,
        actorSystemPort,
        conf,
        securityManager
      )._1
    }

  // Figure out which port Akka actually bound to in casethe original port is 0 or occupied.
  // In the non-driver case, the RPCenv's address may be null since it may not be listening
  // for incoming connections.
  //初始化RpcEnv及ActorSystem之后，port端口变成rpcEnv对应的端口

if (isDriver) {
    conf.set("spark.driver.port", rpcEnv.address.port.toString)
  } else if (rpcEnv.address != null) {
    conf.set("spark.executor.port", rpcEnv.address.port.toString)
  }

  // Create an instance of the class with the given name,possibly initializing it with our conf
  def instantiateClass[T](className: String): T = {
    val cls= Utils.classForName(className)
    // Look for a constructor taking a SparkConf and aboolean isDriver, then one taking just SparkConf, then one taking no arguments
    try {
      //如果构造方法只有一个参数就进入catch生成对应的实例
      cls.getConstructor(classOf[SparkConf], java.lang.Boolean.TYPE)
        .newInstance(conf, new java.lang.Boolean(isDriver))
        .asInstanceOf[T]
    } catch {
      case _:NoSuchMethodException =>
        try {
          cls.getConstructor(classOf[SparkConf]).newInstance(conf).asInstanceOf[T]
        } catch {
          case _:NoSuchMethodException =>
            cls.getConstructor().newInstance().asInstanceOf[T]
        }
    }
  }

  // Create an instance of the class named by the givenSparkConf property, or defaultClassName
  // if the property is not set, possiblyinitializing it with our conf
  def instantiateClassFromConf[T](propertyName: String, defaultClassName: String): T = {
    instantiateClass[T](conf.get(propertyName, defaultClassName))
  }
  //spark如果第一个要优化的地方就是使用Kryo进行网终传输，会比使用java提升10倍以上
  val serializer= instantiateClassFromConf[Serializer](
    "spark.serializer", "org.apache.spark.serializer.JavaSerializer")
  logDebug(s"Using serializer: ${serializer.getClass}")

  val closureSerializer= instantiateClassFromConf[Serializer](
    "spark.closure.serializer", "org.apache.spark.serializer.JavaSerializer")

  def registerOrLookupEndpoint(
      name: String, endpointCreator: => RpcEndpoint):
    RpcEndpointRef = {
    //如果是driver就将BlockManagerMasterEndpoint注到RpcEnv中，如果不是就从RpcEnv中检索出来
    if (isDriver){
      logInfo("Registering " + name)
      rpcEnv.setupEndpoint(name, endpointCreator)
    } else {
      RpcUtils.makeDriverRef(name, conf, rpcEnv)
    }
  }
//查看spark-core_20: MapOutputTrackerMaster、MapOutputTracker、MapOutputTrackerMasterEndpoint、MapStatus、MetadataCleaner的初始化源码分析
  val mapOutputTracker= if (isDriver) {
    /* MapOutputTrackerMaster属于driver，这里使用TimeStampedHashMap来跟踪 map的输出信息，也可以将旧信息进行ttl
      * 一、MapOutputTracker的作用
      * 1，获得mapper的输入信息，方便reducer取得对应的信息
      * 2，每个mapper和reducer都有自己的唯一标识mapperid,reducerId
      * 3,每个reducer可以对应多个map的输入，reducer会去取每个map中的Block,这个过程称为shuffle,每个shuffle也对应shuffleId
      */
    new MapOutputTrackerMaster(conf)
  } else {
    //是运行在executor中的
    new MapOutputTrackerWorker(conf)
  }

  // Have to assign trackerActor after initialization asMapOutputTrackerActor
  // requires the MapOutputTracker itself
  //初始化MapOutputTracker需要给的成员trackerEndpoint进行赋MapOutputTrackerMasterEndpoint，MapOutputTracker.ENDPOINT_NAME的值：MapOutputTracker
  mapOutputTracker.trackerEndpoint = registerOrLookupEndpoint(MapOutputTracker.ENDPOINT_NAME,
    new MapOutputTrackerMasterEndpoint(
      rpcEnv, mapOutputTracker.asInstanceOf[MapOutputTrackerMaster], conf))

  // Let the user specify short names for shuffle managers
  //使用sort shuffle做为mapper和reducer网络传输

//（查看spark-core_21:SortShuffleManager的初始化源码分析）        
  val shortShuffleMgrNames= Map(
    "hash" -> "org.apache.spark.shuffle.hash.HashShuffleManager",
    "sort" -> "org.apache.spark.shuffle.sort.SortShuffleManager",
    "tungsten-sort" -> "org.apache.spark.shuffle.sort.SortShuffleManager")
  val shuffleMgrName= conf.get("spark.shuffle.manager", "sort")
  val shuffleMgrClass= shortShuffleMgrNames.getOrElse(shuffleMgrName.toLowerCase, shuffleMgrName)
  val shuffleManager= instantiateClass[ShuffleManager](shuffleMgrClass)

  // 这个配制针对1.5版本之前遗留模式，它会将堆空间严格地分割成固定大小的区域，如果开启之后。 

不配制spark.shuffle.memoryFraction该参数，会有oom的风险。默认是不开启
  val useLegacyMemoryManager = conf.getBoolean("spark.memory.useLegacyMode", false)

  val memoryManager: MemoryManager =
    if (useLegacyMemoryManager) { //默认是不开启静态内存管理
      new StaticMemoryManager(conf, numUsableCores)
    } else {
      //使用最新的UnifiedMemoryManager内存管理模型，即统一内存管理模型
      UnifiedMemoryManager(conf, numUsableCores)
    }
  /*
      blockTransferService默认为NettyBlockTransferService

      （可以配置属相spark.shuffle.blockTransferService使用NioBlockTransferService）
      ,它使用Netty驱动的网络应用框架，提供web服务及客户端，获取远程节点上的Block集合。
   */
  val blockTransferService = new NettyBlockTransferService(conf, securityManager, numUsableCores)
 //diver上的BlockManagerMaster负责对Executor上的BlockManager进行管理
 //registerOrLookupEndpoint方法如果在driver节点上就将BlockManagerMasterEndpoint注到RpcEnv中，

如果不是就从RpcEnv中检索出来
  val blockManagerMaster = new BlockManagerMaster(registerOrLookupEndpoint(
    BlockManagerMaster.DRIVER_ENDPOINT_NAME,
    new BlockManagerMasterEndpoint(rpcEnv, isLocal, conf, listenerBus)),
    conf, isDriver)

  // NB: blockManager is not valid until initialize() is called later.
  //在sparkEnv初始化时会将BlockManager也初始化出来，同时将NettyBlockTransferService

这个Block传输服务，BlockManager只是负责管理所在Executor上的Block
  //Driver上的BlockManagerMaster对于存在Executor上的BlockManager进行统一管理
  val blockManager = new BlockManager(executorId, rpcEnv, blockManagerMaster,
    serializer, conf, memoryManager, mapOutputTracker, shuffleManager,
    blockTransferService, securityManager, numUsableCores)
// BroadcastManager用于将配置信息和序列化后的RDD,job以及ShuffleDependency等信息在本地存储。

  如果为了容灾，也会复制到其他节点上。
  val broadcastManager = new BroadcastManager(isDriver, conf, securityManager)
  //负责将RDD分区内容传递给BlockManager，并确保节点不会一次加载RDD的两个副本。
  //CacheManager是管理缓存数据，缓存可以是基于内存的缓存，也可以是基于磁盘的缓存；
  val cacheManager = new CacheManager(blockManager)

  val metricsSystem = if (isDriver) {
    // Don't start metrics system right now for Driver.
    // We need to wait for the task scheduler to give us an app ID.
    // Then we can start the metrics system.
    MetricsSystem.createMetricsSystem("driver", conf, securityManager)
  } else {
    // We need to set the executor ID before the MetricsSystem is created because sources and
    // sinks specified in the metrics configuration file will want to incorporate this executor's
    // ID into the metrics they report.
    conf.set("spark.executor.id", executorId)
    val ms = MetricsSystem.createMetricsSystem("executor", conf, securityManager)
    ms.start()
    ms
  }

  // Set the sparkFiles directory, used when downloading dependencies.  In local mode,
  // this is a temporary directory; in distributed mode, this is the executor's current working
  // directory.
  //设置spark下载依赖目录，在本地模式下，这是一个临时目录; 在分布式模式下，

是executor当前的工作目录。
  val sparkFilesDir: String = if (isDriver) {
    Utils.createTempDir(Utils.getLocalDir(conf), "userFiles").getAbsolutePath
  } else {
    "."
  }
  //决定是否将task的输出放到hdfs中，该类会在dirver或executor上被实例化，
 
  val outputCommitCoordinator = mockOutputCommitCoordinator.getOrElse {
    new OutputCommitCoordinator(conf, isDriver)
  }
  //如果是driver会将OutputCommitCoordinatorEndpoint注册到RpcEnv中，

如果execute则会从RpcEnv中取OutputCommitCoordinatorEndpoint
  val outputCommitCoordinatorRef = registerOrLookupEndpoint("OutputCommitCoordinator",
    new OutputCommitCoordinatorEndpoint(rpcEnv, outputCommitCoordinator))
  outputCommitCoordinator.coordinatorRef = Some(outputCommitCoordinatorRef)

  val envInstance = new SparkEnv(
    executorId,
    rpcEnv,
    actorSystem,
    serializer,
    closureSerializer,
    cacheManager,
    mapOutputTracker,
    shuffleManager,
    broadcastManager,
    blockTransferService,
    blockManager,
    securityManager,
    sparkFilesDir,
    metricsSystem,
    memoryManager,
    outputCommitCoordinator,
    conf)

  // Add a reference to tmp dir created by driver, we will delete this tmp dir when stop() is
  // called, and we only need to do it for driver. Because driver may run as a service, and if we
  // don't delete this tmp dir when sc is stopped, then will create too many tmp dirs.
  if (isDriver) {
    envInstance.driverTmpDirToDelete = Some(sparkFilesDir)
  }

  envInstance
}