scala-spark-worldcount

分子美食家

于 2020-09-01 00:25:54 发布

阅读量119

点赞数

分类专栏： scala spark

本文链接：https://blog.csdn.net/weixin_40083227/article/details/108332747

版权

spark 同时被 2 个专栏收录

18 篇文章 0 订阅

订阅专栏

scala

12 篇文章 0 订阅

订阅专栏

import org.apache.spark._
import org.apache.spark.rdd.RDD.rddToOrderedRDDFunctions
import org.apache.spark.rdd.RDD.rddToPairRDDFunctions

object SparkWordCount {
  def main(args: Array[String]) {

    if (args.length < 1) {
      System.err.println("Usage: <file>")
      System.exit(1)
    }

    val conf = new SparkConf()
    conf.setAppName("SparkWordCount")

    val sc = new SparkContext(conf)

    val rdd = sc.textFile(args(0))

    rdd.flatMap(_.split(" ")).map((_, 1)).reduceByKey(_ + _).map(x => (x._2, x._1)).sortByKey(false).map(x => (x._2, x._1)).saveAsTextFile(args(1))

    sc.stop
  }
}