Java Spark算子：distinct

最新推荐文章于 2022-07-22 11:52:31 发布

默默倾听全世界

最新推荐文章于 2022-07-22 11:52:31 发布

阅读量266

点赞数

本文链接：https://blog.csdn.net/weixin_44576373/article/details/104396402

版权

Java 同时被 2 个专栏收录

23 篇文章 0 订阅

订阅专栏

Spark

21 篇文章 0 订阅

订阅专栏

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;

import java.util.Arrays;
import java.util.List;

/**
 * distinct（）算子
 * 去除RDD的重复元素。
 *
 */
public class DistinctDemo {

    public static void main(String[] args) {

        SparkConf conf = new SparkConf().setMaster("local").setAppName("spark");
        JavaSparkContext sc  = new JavaSparkContext(conf);
        List<String> list = Arrays.asList("a","b","c","a","b","c","d");
        JavaRDD<String> javaRDD = sc.parallelize(list);

        //distinct算子：去重
        JavaRDD<String> reduce = javaRDD.distinct();
        System.err.println(reduce.collect());
    }

}