scala版wordcount

需求

用scala实现计算wordcount

完整实现

package core

import org.apache.spark.{SparkConf, SparkContext}

/**
  * 
  */
object BasicWordCountApp {
  def main(args: Array[String]): Unit = {
    //获得SparkConf
    var conf = new SparkConf().setAppName("BasicWordCountApp").setMaster("local")

    //获得SparkContext
    val sc = new SparkContext(conf);

    //加载本地文件，准备lineRDD
    val lines = sc.textFile("D:/WordCount.txt")

    //将每行数据根据空格分割，得到wordRDD
    val wordsRDD = lines.flatMap(_.split(" "))

    //转换为pairRDD
    val pairRDD = wordsRDD.map((_,1))

    //聚合
    val wcRDD = pairRDD.reduceByKey(_+_)

    //action操作，打印
    wcRDD.foreach(t => println(t))
  }
}