spark学习记录（八、广播变量和累加器）

最新推荐文章于 2020-06-16 15:31:12 发布

在知识的海洋中遨游

最新推荐文章于 2020-06-16 15:31:12 发布

阅读量138

点赞数

分类专栏： spark

本文链接：https://blog.csdn.net/qq_33283652/article/details/85772540

版权

spark 专栏收录该内容

18 篇文章 1 订阅

订阅专栏

一、广播变量

public class JavaExample {
    public static void main(String[] args) {
        SparkConf conf = new SparkConf();
        conf.setMaster("local").setAppName("JavaExample");
        JavaSparkContext sc = new JavaSparkContext(conf);

        final List<String> list = Arrays.asList("hello world", "hello spark");
        //广播变量
        final Broadcast<List<String>> broadcast = sc.broadcast(list);
        JavaRDD<String> rdd1 = sc.parallelize(Arrays.asList("hello world", "hello spark", "hello java"));
        JavaRDD<String> rdd2 = rdd1.filter(new Function<String, Boolean>() {
            public Boolean call(String line) throws Exception {
                return !broadcast.value().contains(line);
            }
        });
        rdd2.foreach(new VoidFunction<String>() {
            public void call(String s) throws Exception {
                System.out.println(s);
            }
        });
    }
}

二、累加器

object ScalaExample {
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setMaster("local").setAppName("ScalaExample")
    val sc = new SparkContext(conf);
    val rdd1 = sc.textFile("C://words.txt")
//    累加器
    val accumulator = sc.accumulator(0);
    rdd1.map(line =>{
      accumulator.add(1)
      println(accumulator)
      line
    }).collect()
    println(" i = " +accumulator.value)
  }
}