object leijiaqi {
def main(args: Array[String]): Unit = {
//基本配置
val conf = new SparkConf().setAppName("myRddLearn").setMaster("local[*]")
val spark = SparkSession.builder().config(conf).getOrCreate()
val initRdd = spark.sparkContext.makeRDD(List("hello", "word", "spark", "hive", "hbase"))
val accumulator = new MyAccumulator
spark.sparkContext.register(accumulator)
initRdd.foreach({
case x => {
accumulator.add(x)
}
})
println(accumulator.value)
spark.stop()
}
}
/**
* TODO: 自定义累加器
* 继承AccumulatorV2,实现方法
* 使用的时候需要注册到spark中去
*/
class MyAccumulator extends AccumulatorV2[String, util.ArrayList[String]] {
val list = new util.ArrayList[String]()
//判断是否为空
override def isZero: Boolean = list.isEmpty
//复制累加器
override def copy(): AccumulatorV2[String, util.ArrayList[String]] = {
new MyAccumulator()
}
//重置累加器
override def reset(): Unit = {
list.clear()
}
//增加一个元素
override def add(v: String): Unit = {
if (v.contains("h")) {
list.add(v)
}
}
//合并每个分区累加器
override def merge(other: AccumulatorV2[String, util.ArrayList[String]]): Unit = {
list.addAll(other.value)
}
//返回累加器的值
override def value: util.ArrayList[String] = list
}
Spark自定义累加器
最新推荐文章于 2024-02-23 20:05:17 发布