spark2.0源码阅读 剖析sparkConf

sparkConf存储结构

//通过安全的HashMap保存所有配置
private val settings = new ConcurrentHashMap[String, String]()

sparkConf会加载系统变量spark.打头的属性，具体实现如下

//加载默认配置,loadDefaults默认为true
if (loadDefaults) {
  loadFromSystemProperties(false)
}

private[spark] def loadFromSystemProperties(silent: Boolean): SparkConf = {
  // Load any spark.* system properties
  // 调用了Utils工具类[2] 的getSystemProperties方法，其作用为获取系统的键值对属性。
  for ((key, value) <- Utils.getSystemProperties if key.startsWith("spark.")) {
    set(key, value, silent)
  }
  this
}

/** Set a configuration variable. */
def set(key: String, value: String): SparkConf = {
  set(key, value, false)
}

重写的set方法，主要是加了静默模式，以及空值判断

private[spark] def set(key: String, value: String, silent: Boolean): SparkConf = {
  if (key == null) {
    throw new NullPointerException("null key")
  }
  if (value == null) {
    throw new NullPointerException("null value for " + key)
  }
  if (!silent) {
    //如果是静默模式，就打印抑制配置的warning日志
    logDeprecationWarning(key)
  }
  //加入配置文件
  settings.put(key, value)
  this
}

克隆SparkConf配置

class SparkConf(loadDefaults: Boolean) extends Cloneable with Logging with Serializable {

  import SparkConf._

具体clone方法的实现

override def clone: SparkConf = {
  val cloned = new SparkConf(false)
  settings.entrySet().asScala.foreach { e =>
    cloned.set(e.getKey(), e.getValue(), true)
  }
  cloned
}

后面很多地方会调用到。具体到了再说