scala与java编写WordCount

scala编写WordCount

object WordCount {
  def main(args: Array[String]): Unit = {
    //创建SparkContext,只有使用SparkContext才可以向集群申请资源,才可以创建RDD
    val conf = new SparkConf().setAppName("WordCount")
    val sc = new SparkContext(conf)

    //第一步创建RDD:指定[以后]从HDFS中读取数据创建RDD  最小分区数

    val lines: RDD[String] = sc.textFile(args(0))

    //对数据切分压平
    val words = lines.flatMap(_.split(" "))

    //将单词和1组合
    val wordAndOne = words.map((_, 1))

    //聚合(现在分区内局部聚合,在全局聚合)
    val reduced = wordAndOne.reduceByKey(_ + _)

    //排序
    val sorted = reduced.sortBy(_._2, false)

    //将计算完的结果保存到ＨＤＦＳ中
    sorted.saveAsTextFile(args(1))

    //释放资源
    sc.stop()

  }
}

 java编写WordCount

public class JavaWordCount {
    public static void main(String[] args) {

        //创建sparkContext 因为只有使用SparkContext才能调用集群资源,才能创建RDD
        SparkConf conf = new SparkConf().setAppName("javaWordCount");
        JavaSparkContext jsc = new JavaSparkContext(conf);

        //创建RDD  读取数据用                          最小分区数
        JavaRDD<String> lines = jsc.textFile(args[0]);

        //切分压平
        JavaRDD<String> words = lines.flatMap(new FlatMapFunction<String, String>() {
            @Override
            public Iterator<String> call(String s) throws Exception {
                return Arrays.stream(s.split(" ")).iterator();
            }
        });

        //将单词和1 组合成元组
        JavaPairRDD<String, Integer> wordAndOne = words.mapToPair(new PairFunction<String, String, Integer>() {
            @Override
            public Tuple2<String, Integer> call(String s) throws Exception {
                return Tuple2.apply(s, 1);
            }
        });

        //聚合
        JavaPairRDD<String, Integer> reduced = wordAndOne.reduceByKey(new Function2<Integer, Integer, Integer>() {
            @Override
            public Integer call(Integer v1, Integer v2) throws Exception {
                return v1 + v2;
            }
        });

        //交换顺序,排序只能按key排,但key时单词,所以与数字交换位置
        JavaPairRDD<Integer, String> swap = reduced.mapToPair(new PairFunction<Tuple2<String, Integer>, Integer, String>() {
            @Override
            public Tuple2<Integer, String> call(Tuple2<String, Integer> tp) throws Exception {
                //swap方法就是交换位置
                return tp.swap();
            }
        });
        //排序
        JavaPairRDD<Integer, String> sorted = swap.sortByKey(false);

        //在交换顺序,因为打印结果想要的时单词在前,数字在后
        JavaPairRDD<String, Integer> word = sorted.mapToPair(new PairFunction<Tuple2<Integer, String>, String, Integer>() {
            @Override
            public Tuple2<String, Integer> call(Tuple2<Integer, String> tp) throws Exception {
                return tp.swap();
            }
        });
        //保存到Hdfs中
        word.saveAsTextFile(args[1]);

        jsc.stop();


    }
}