spark性能调优之RDD缓存模型

/**
   * Mark this RDD for persisting using the specified level.
   *
   * @param newLevel the target storage level
   * @param allowOverride whether to override any existing level with the new one
   * 这个方法适用于之前我们设置过了RDD的缓存级别，现在想要修改RDD的缓存级别的情况，只需要把allowOverride设置为true
   */
  private def persist(newLevel: StorageLevel, allowOverride: Boolean): this.type = {
    // TODO: Handle changes of StorageLevel
    //这一段程序也解释了上面第二种方法的第一个特殊情况为什么不会报错
    if (storageLevel != StorageLevel.NONE && newLevel != storageLevel && !allowOverride) {
      throw new UnsupportedOperationException(
        "Cannot change storage level of an RDD after it was already assigned a level")
    }
    // If this is the first time this RDD is marked for persisting, register it
    // with the SparkContext for cleanups and accounting. Do this only once.
    if (storageLevel == StorageLevel.NONE) {
      sc.cleaner.foreach(_.registerRDDForCleanup(this))
      sc.persistRDD(this)
    }
    storageLevel = newLevel
    this
  }

 

StorageLevel这个类里面设置了RDD的各种缓存级别，总共有12种，其实是它的多个构造参数的组合形成的，先看一下它的相关构造参数，源码如下：

@DeveloperApi
class StorageLevel private(
    private var _useDisk: Boolean,    //是否使用磁盘
    private var _useMemory: Boolean,  //是否使用内存
    private var _useOffHeap: Boolean, //是否使用堆外内存
    private var _deserialized: Boolean, //是否反序列化
    private var _replication: Int = 1)  //备份因子，默认为1
  extends Externalizable {

 

 

而它设定的12种级别就是这些参数取值的组合：

object StorageLevel {
  val NONE = new StorageLevel(false, false, false, false)
  val DISK_ONLY = new StorageLevel(true, false, false, false)
  val DISK_ONLY_2 = new StorageLevel(true, false, false, false, 2)
  val MEMORY_ONLY = new StorageLevel(false, true, false, true)
  val MEMORY_ONLY_2 = new StorageLevel(false, true, false, true, 2)
  val MEMORY_ONLY_SER = new StorageLevel(false, true, false, false)
  val MEMORY_ONLY_SER_2 = new StorageLevel(false, true, false, false, 2)
  val MEMORY_AND_DISK = new StorageLevel(true, true, false, true)
  val MEMORY_AND_DISK_2 = new StorageLevel(true, true, false, true, 2)
  val MEMORY_AND_DISK_SER = new StorageLevel(true, true, false, false)
  val MEMORY_AND_DISK_SER_2 = new StorageLevel(true, true, false, false, 2)
  val OFF_HEAP = new StorageLevel(true, true, true, false, 1)

 

转载于:https://www.cnblogs.com/sen-2017/p/11451959.html