Scala实现经典案例：WordCount

Word Count 案例

scala版本：2.12

普通 Word Count 案例

需求

单词计数：将集合中出现的相同的单词，进行计数，取计数排名前三的结果

代码实现

package com.kuber.chapter7

object CommonWordCount {
  def main(args: Array[String]): Unit = {
    val list: List[String] = List(
      "hello world",
      "hello scala",
      "hello spark",
      "hello spark from scala"
    )

    //1.对字符串进行切分
    //    val wordList1: List[Array[String]] = list.map(_.split(" "))
    //    val wordList2: List[String] = wordList1.flatten
    //    println(wordList2)
    val wordList: List[String] = list.map(_.split(" ")).flatten
    println(wordList)

    //2.相同的单词进行分组
    val groupMap: Map[String, List[String]] = wordList.groupBy(word => word)
    println(groupMap)

    //3.对分组之后的list取长度得到每个单词的个数
    val wordCountMap: Map[String, Int] = groupMap.map(kv => (kv._1, kv._2.length))
    println(wordCountMap)

    //4.将map转换为list，并排序取前3
    val sortList: List[(String, Int)] = wordCountMap.toList.sortWith(_._2 > _._2)
    println(sortList)

    //5.取前三名
    val resList: List[(String, Int)] = sortList.take(3)
    println(resList)

  }
}

每一步的执行结果

复杂 Word Count 案例

需求

数据类型发生改变，为

val tupleList: List[(String, Int)] = List(
  ("hello world", 1),
  ("hello scala", 2),
  ("hello spark", 3),
  ("hello spark from scala", 4)
)

代码实现

package com.kuber.chapter7

object ComplexWordCount {
  def main(args: Array[String]): Unit = {
    val tupleList: List[(String, Int)] = List(
      ("hello world", 1),
      ("hello scala", 2),
      ("hello spark", 3),
      ("hello spark from scala", 4)
    )
    //思路一：直接展开为普通版本
    val newList: List[String] = tupleList.map(
      kv => {
        ((kv._1.trim) + " ") * kv._2
      }
    )
    println(newList)
    //接下来操作和普通版本完全一致
    //切割，扁平化打散
    val flattenList: List[String] = newList.map(_.split(" ")).flatten
    println(flattenList)
    //相同的单词进行分组
    val groupMap: Map[String, List[String]] = flattenList.groupBy(word => word)
    println(groupMap)
    //统计个数
    val resMap: Map[String, Int] = groupMap.map(kv => (kv._1, kv._2.length))
    //将map转化成list,并按从大到校排列
    val resTuple: List[(String, Int)] = resMap.toList.sortWith(_._2 > _._2)
    println(resTuple)
    //取前三名
    println(resTuple.take(3))

    println("========================================")

    //思路二：直接基于预统计的结果进行转换
    //1.将字符串打散为单词，并结合对应的个数包装成二元组
    val preCountList: List[(String, Int)] = tupleList.flatMap(
      tuple => {
        val strings: Array[String] = tuple._1.split(" ")
        strings.map(word => (word, tuple._2))
      }
    )
    println(preCountList)
    //2.对二元组按照单词再进行分组
    val preGroupMap: Map[String, List[(String, Int)]] = preCountList.groupBy(tuple => tuple._1)
    println(preGroupMap)
    //3.叠加每个单词预统计的个数值
    val resMap2: Map[String, Int] = preGroupMap.mapValues(
      tupleList => tupleList.map(_._2).sum
    )
    println(resMap2)
    //4.将map转换成list并排序输出
    val resList2: List[(String, Int)] = resMap2.toList.sortWith(_._2 > _._2)
    println(resList2)
    //5.取前三名
    println(resList2.take(3))
  }
}

每一步的执行结果