Scala 词频统计WordCount实例

1. 实现一个词频统计的demo。

data目录是工程下与src同级的目录。

import java.io.File
import scala.io.Source

object WordCount {
  def main(args: Array[String]): Unit = {
    val dir_file = new File("data")
    val file_list = dir_file.listFiles().toList
    for (f <- file_list) println(f)
    val words_map = scala.collection.mutable.Map[String,Int]()
    file_list.foreach(
      Source.fromFile(_).getLines()     // 处理每一个文件
        .foreach(_.split(" ")    // 处理每一行
        .foreach(word => {
          if(words_map.contains(word)){
            words_map(word) += 1
          } else {
            words_map += (word -> 1)
          }
        }))
    )

    for(kv <- words_map) println(kv._1 + ": " + kv._2)
  }
}

2. 使用rdd进行优化

package com.zz.spark

import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{Dataset, SparkSession}

object WordCount {
  def main(args: Array[String]): Unit = {
    val spark = SparkSession
      .builder()
      .appName("wc")
      .master("local")
      .getOrCreate()

    import spark.implicits._
    var inputPath = "aa.txt"
    val dataRDD: Dataset[String] = spark.read.textFile(inputPath)
    val wordAndOne: Dataset[(String, Int)] = dataRDD
      .flatMap(line => line.split("\\s+"))
      .map((_, 1))

    val result: RDD[(String, Int)] = wordAndOne
      .rdd
      .reduceByKey(_ + _).sortBy(_._2)
    result.collect().foreach(println)
    spark.close()
  }
}