需求
用scala实现计算wordcount
完整实现
package core
import org.apache.spark.{SparkConf, SparkContext}
/**
*
*/
object BasicWordCountApp {
def main(args: Array[String]): Unit = {
//获得SparkConf
var conf = new SparkConf().setAppName("BasicWordCountApp").setMaster("local")
//获得SparkContext
val sc = new SparkContext(conf);
//加载本地文件,准备lineRDD
val lines = sc.textFile("D:/WordCount.txt")
//将每行数据根据空格分割,得到wordRDD
val wordsRDD = lines.flatMap(_.split(" "))
//转换为pairRDD
val pairRDD = wordsRDD.map((_,1))
//聚合
val wcRDD = pairRDD.reduceByKey(_+_)
//action操作,打印
wcRDD.foreach(t => println(t))
}
}