Spark入门之JavaLambdaWordCount

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import scala.Tuple2;

import java.util.Arrays;

public class JavaLambdaWordCount {
    public static void main(String[] args) {

        if (args.length != 2) {
            System.out.println("Usage:clw.spark.day01.ScalaWordCount    <srcPath>,<desPath>");

            System.exit(-1);
        }


        //创建SparkContext
        SparkConf javaLambdaWordCount = new SparkConf().setAppName("JavaLambdaWordCount").setMaster("local[*]");

        JavaSparkContext jsc = new JavaSparkContext(javaLambdaWordCount);

        //指定以后从哪里读取数据
        JavaRDD<String> data = jsc.textFile(args[0]);

        //切分压平
        JavaRDD<String> words = data.flatMap(line -> Arrays.asList(line.split(",")).iterator());

        //将单词和1组合
        JavaPairRDD<String, Integer> wordAndOne = words.mapToPair(w -> new Tuple2<>(w, 1));

        //聚合
        JavaPairRDD<String, Integer> reduced = wordAndOne.reduceByKey((m, n) -> m + n);

        //调换顺序
        JavaPairRDD<Integer, String> swaped = reduced.mapToPair(re -> re.swap());

        //排序
        JavaPairRDD<Integer, String> sorted = swaped.sortByKey(false);

        //调换顺序
        JavaPairRDD<String, Integer> result = sorted.mapToPair(so -> so.swap());

        //将结果存入指定的文件
        result.saveAsTextFile(args[1]);


        jsc.stop();
    }
}