1. 实现一个词频统计的demo。
data目录是工程下与src同级的目录。
import java.io.File
import scala.io.Source
object WordCount {
def main(args: Array[String]): Unit = {
val dir_file = new File("data")
val file_list = dir_file.listFiles().toList
for (f <- file_list) println(f)
val words_map = scala.collection.mutable.Map[String,Int]()
file_list.foreach(
Source.fromFile(_).getLines() // 处理每一个文件
.foreach(_.split(" ") // 处理每一行
.foreach(word => {
if(words_map.contains(word)){
words_map(word) += 1
} else {
words_map += (word -> 1)
}
}))
)
for(kv <- words_map) println(kv._1 + ": " + kv._2)
}
}
2. 使用rdd进行优化
package com.zz.spark
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{Dataset, SparkSession}
object WordCount {
def main(args: Array[String]): Unit = {
val spark = SparkSession
.builder()
.appName("wc")
.master("local")
.getOrCreate()
import spark.implicits._
var inputPath = "aa.txt"
val dataRDD: Dataset[String] = spark.read.textFile(inputPath)
val wordAndOne: Dataset[(String, Int)] = dataRDD
.flatMap(line => line.split("\\s+"))
.map((_, 1))
val result: RDD[(String, Int)] = wordAndOne
.rdd
.reduceByKey(_ + _).sortBy(_._2)
result.collect().foreach(println)
spark.close()
}
}