Scala实现wordcount

最新推荐文章于 2023-11-20 15:31:37 发布

迷途小羔羊。

最新推荐文章于 2023-11-20 15:31:37 发布

阅读量232

点赞数

分类专栏： spark 文章标签： scala spark

本文链接：https://blog.csdn.net/Zzh1110/article/details/103721856

版权

spark 专栏收录该内容

4 篇文章 0 订阅

订阅专栏

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

object WordCount {
  def main(args: Array[String]): Unit = {
    val config: SparkConf = new SparkConf().setMaster("local[*]").setAppName("WordCount")

    val sc = new SparkContext(config)
    //    println(sc)
    val lines: RDD[String] = sc.textFile("in/word.txt")
    val words: RDD[String] = lines.flatMap(x=>x.split(" "))
    val wordToOne: RDD[(String, Int)] = words.map(x=>(x,1))
    val wordToSum: RDD[(String, Int)] = wordToOne.reduceByKey((x,y)=>(x+y))

    val result: Array[(String, Int)] = wordToSum.collect()
    for(word <- result) println(word)
    // lines.flatMap(_.split(" ")).map((_, 1)).reduceByKey(_+_).collect().foreach(println)
  }
}