object WordCount {
def main(args: Array[String]): Unit = {
val lines = Array("hello tom hello jerry", "hello kitty hello tom", "hello tom tom")
//按空格切割行成词,形成RDD(Array(),Array(),...),在flatmap压平成RDD(...)
val words: Array[String] = lines.flatMap(l => l.split(" "))
//对每个单词映射成元组,value为1
val wordAndOne: Array[(String, Int)] = words.map(w => (w, 1))
//按照单词分组,成RDD((word,Array(...)),...)
val grouped: Map[String, Array[(String, Int)]] = wordAndOne.groupBy(t => t._1)
//对每个单词计算value
val wordAndCounts: Map[String, Int] = grouped.map(t => (t._1, t._2.length))
//按照value排序
val result: List[(String, Int)] = wordAndCounts.toList.sortBy(t => -t._2)
//打印结果
print(result.toBuffer)
}
}
spark编写wordcount程序
最新推荐文章于 2022-05-28 22:20:15 发布