Spark自定义累加器

object leijiaqi {

  def main(args: Array[String]): Unit = {

    //基本配置
    val conf = new SparkConf().setAppName("myRddLearn").setMaster("local[*]")
    val spark = SparkSession.builder().config(conf).getOrCreate()

    val initRdd = spark.sparkContext.makeRDD(List("hello", "word", "spark", "hive", "hbase"))
    val accumulator = new MyAccumulator
    spark.sparkContext.register(accumulator)
    initRdd.foreach({
      case x => {
        accumulator.add(x)
      }
    })
    println(accumulator.value)
    spark.stop()

  }

}

/**
  * TODO： 自定义累加器
  * 继承AccumulatorV2,实现方法
  * 使用的时候需要注册到spark中去
  */
class MyAccumulator extends AccumulatorV2[String, util.ArrayList[String]] {

  val list = new util.ArrayList[String]()

  //判断是否为空
  override def isZero: Boolean = list.isEmpty

  //复制累加器
  override def copy(): AccumulatorV2[String, util.ArrayList[String]] = {
    new MyAccumulator()
  }

  //重置累加器
  override def reset(): Unit = {
    list.clear()
  }

  //增加一个元素
  override def add(v: String): Unit = {
    if (v.contains("h")) {
      list.add(v)
    }
  }

  //合并每个分区累加器
  override def merge(other: AccumulatorV2[String, util.ArrayList[String]]): Unit = {
    list.addAll(other.value)
  }

  //返回累加器的值
  override def value: util.ArrayList[String] = list
}