RDD-缓存

  /**
   * 统计访问次数最多的ip
   * None     0.244744 s
   * cache    0.126583 s
   * persist  0.132369 s
   * 
   * cache 底层 调用的 是 persist
   * persist 默认的 缓存级别是 StorageLevel.MEMORY_ONLY
   */
  @Test
  def prepare(): Unit = {
    val resouce: RDD[String] = sc.textFile("dataset/access.log")
    val result: RDD[(String, Int)] = resouce.map(item => (item.split(" ")(0), 1))
      .reduceByKey((curr, agg) => curr + agg)
      .persist(StorageLevel.MEMORY_ONLY)

    val max: Array[(String, Int)] = result.sortBy(item => item._2, ascending = false).take(1)
    val min: Array[(String, Int)] = result.sortBy(item => item._2, ascending = true).take(1)
    max.foreach(println(_))
    min.foreach(println(_))
  }

源码 RDD.scala

  /**
   * Set this RDD's storage level to persist its values across operations after the first time
   * it is computed. This can only be used to assign a new storage level if the RDD does not
   * have a storage level set yet. Local checkpointing is an exception.
   */
  def persist(newLevel: StorageLevel): this.type = {
    if (isLocallyCheckpointed) {
      // This means the user previously called localCheckpoint(), which should have already
      // marked this RDD for persisting. Here we should override the old storage level with
      // one that is explicitly requested by the user (after adapting it to use disk).
      persist(LocalRDDCheckpointData.transformStorageLevel(newLevel), allowOverride = true)
    } else {
      persist(newLevel, allowOverride = false)
    }
  }

  /**
   * Persist this RDD with the default storage level (`MEMORY_ONLY`).
   */
  def persist(): this.type = persist(StorageLevel.MEMORY_ONLY)

  /**
   * Persist this RDD with the default storage level (`MEMORY_ONLY`).
   */
  def cache(): this.type = persist()