wordcount12种实现方式

wordcount12种实现方式

数据准备

hello world
dog fish
hadoop 
spark
hello world
dog fish
hadoop 
spark
hello world
dog fish
hadoop `在这里插入代码片`
spark

代码

import org.apache.spark.{SparkConf, SparkContext}

import scala.collection.mutable

object methods_1_CountByKey {
  def main(args: Array[String]): Unit = {
    val conf: SparkConf = new SparkConf().setMaster("local[*]").setAppName("methods_1_CountByKey")
    val sc = new SparkContext(conf)
    val lines = sc.textFile("G:\\大数据\\wordcount.txt")
    val word = lines.flatMap(_.split(" "))
    val wordkv = word.map((_,1))

    // TODO:  方法1：mapValues
    println("方法1：mapValues：")
    val wordgroupby = wordkv.groupBy(_._1)
    val resultmapValues = wordgroupby.mapValues(_.size)
    resultmapValues.collect().foreach(print)
    println("")
    println("方法2：CountByvalue：")
    // TODO: 方法2：CountByvalue----不需要变成KV结构的数据形式
    val resultcountByValues = wordkv.countByValue()
//    resultcountByValues.foreach(print)//相同key出现的次数，次数为v，不论之前的数据是k结构还是kv结构，都当成k。
      word.countByValue().foreach(print(_))
    // TODO: 方法3：reduceBykey
    println("")
    println("方法3：reduceBykey ：")
    wordkv.reduceByKey(_+_).collect().foreach(print(_))
    // TODO: 方法4：aggregateBykey
    println("")
    println("方法4：reduceBykey ：")
    wordkv.aggregateByKey(0)(_+_,_+_).collect().foreach(print(_))
    // TODO: 方法5：foldBykey
    println("")
    println("方法5：reduceBykey ：")
    wordkv.foldByKey(0)(_+_).collect().foreach(print)
    // TODO: 方法6：foldBykey
    println("")
    println("方法6：combineBykey ：")
   wordkv.combineByKey(word=>word,(x:Int,y:Int)=> x+y,(x:Int,y:Int)=> x+y).collect().foreach(print)

    // TODO: 方法7：groupByKey+Map(这里不能用groupby只能用groupbykey）
    println("")
    println("方法7：Map+sum")
    wordkv.groupByKey().map(word=>(word._1,word._2.sum)).collect().foreach(print)
    // TODO: 方法8：groupByKey
    println("")
    println("方法8：groupByKey：")
    wordkv.groupByKey().mapValues(_.size).collect().foreach(print(_))
    // TODO: 方法9：reduce
    println("")
    println("方法9：reduce：")
   wordkv.groupByKey().map(x=>(x._1,x._2.reduce((x:Int,y:Int)=>x+y:Int))).collect().foreach(print(_))
    // TODO: 方法10：Aggregate
    println("")
    println("方法10：Aggregate：")



//    val i = wordkv.map(w=>w._2).aggregate(0)(_+_,_+_)
    val rdd5_2 = wordkv.aggregate(Map[String,Int]())(
      (map1,kv) => {
        val k = kv._1
        val v = kv._2
        val newvalue = map1.getOrElse(k,0)+v
        map1.updated(k,newvalue)
      },
      (map1,map2) => {
        map1.foldLeft(map2)((map, kv) => {
          val k: String = kv._1
          val v: Int = kv._2
          val newValue = map.getOrElse(k, 0) + v
          map.updated(k, newValue)
        })
      }
    )
    rdd5_2.foreach(print)

    // TODO: 方法11：CountBykey
    println("")
    println("方法11：CountBykey：")
    wordkv.countByKey().foreach(print(_))

    // TODO: 方法12：fold
    println("")
    println("方法12：fold：")
    word.map(s => mutable.Map(s->1)).fold(mutable.Map[String,Int]())(
      (map1,map2) => {
        map1.foldLeft(map2)(
          ( innerMap, kv) => {
            innerMap(kv._1) = innerMap.getOrElse(kv._1, 0) + kv._2
            innerMap
          }
        )`在这里插入代码片`
      }
    ).foreach(print)
  }
}