SparkCore快速入门-自定义累加器的编写

import org.apache.spark.util.AccumulatorV2
import org.apache.spark.{SparkConf, SparkContext}

/**
 * 自定义累加器
 */
object SparkCore_CustomAccumulator {
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setAppName("test").setMaster("local")
    val sc = new SparkContext(conf)

    val accumulator = new CustomAccumulator
    //注册累加器
    sc.register(accumulator, "acc")

    //利用自定义累加器 ，累加求和
    var li = List(1, 2, 3, 4)
    val liRDD = sc.makeRDD(li)
    liRDD.foreach(i => {
      //调用add()进行累加
      accumulator.add(i)
    })
    //调用value 进行取值
    val result = accumulator.value
    println(result) //10

    sc.stop()
  }

}
//编写自定义累加器类
class CustomAccumulator extends AccumulatorV2[Int, Int] {
  //定义累加变量
  private var _sum: Int = _

  //判断是否为0
  override def isZero: Boolean = _sum == 0

  //创建新的对象
  override def copy(): AccumulatorV2[Int, Int] = {
    val accumulator = new CustomAccumulator
    accumulator._sum = this._sum
    accumulator
  }

  //重置
  override def reset(): Unit = _sum == 0

  //局部聚合累加
  override def add(v: Int): Unit = _sum += v

  //全局聚合累加
  override def merge(other: AccumulatorV2[Int, Int]): Unit = _sum += other.value

  //最终结果
  override def value: Int = _sum
}

可以参照继承了AccumulatorV2的子类进行模仿编写
1.进入AccumulatorV2

2.按Ctrl+h ，弹出下图所示，随便点击一个

3.可以看到跟上述编写基本一致，参考即可