sparkRDD操作之ReduceByKey和GroupByKey比较

sparkRDD操作之ReduceByKey和GroupByKey比较

直接看代码

package cn.sparkdemo.core

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

object TestRDDFuctions {
  def main(args: Array[String]): Unit = {
    //1、创建上下文对象
    val sparkConf: SparkConf = new SparkConf()
    .setAppName(this.getClass.getSimpleName.stripSuffix("$"))
    .setMaster("local[*]")
    val sc = new SparkContext(sparkConf)
    //2、加载数据形成RDD
    val inputRDD: RDD[String] = sc.parallelize(Array(
      "hello me you her",
      "hello you her",
      "hello her",
      "hello"
    ))
    //3、将RDD的长文本拆分成单词
    val wordRDD: RDD[String] = inputRDD.flatMap(_.split("\\s+"))
    //4、继续为每个单词标记为1
    //tupleRDD :[(hello,1),(hello,1),(hello,1),(hello,1),(me,1),(you,1)..........]
    val tupleRDD: RDD[(String, Int)] = wordRDD.map((_, 1))

    //5、用groupByKey来做wordcount
    //groupedRDD:[(hello,[1,1,1,1]),(me,[1]),(you,[1,1]),.......]
    val groupedRDD: RDD[(String, Iterable[Int])] = tupleRDD.groupByKey()
    //（x -> y) 即元组（x,y）
    //wordcountRDD1:[(hello,4),(me,1),(you,2).......]
    val wordcountRDD1: RDD[(String, Int)] = groupedRDD.map(x => x._1 -> x._2.sum)
    //用另一个写法
    val wordcountRDD2: RDD[(String, Int)] = groupedRDD.mapValues(x => x.sum)
    //6、打印结果
    println("groupByKey:")
    wordcountRDD1.foreach(println(_))
    //groupedRDD.foreach(println(_))
    //7、用reduceByKey和foldByKey来做wordcount
    //tupleRDD 以hello为例， 有(hello,1) (hello,1) (hello,1) (hello,1)
    //reduceBYKey , 其实是先groupByKey再reduce
    //先ByKey，变成了 hello -> [1,1,1,1]
    //再reduce，对[1,1,1,1]进行聚合比如 _+_ ， 变成了4
    //最后总体变成了hello -> 4
    //上面2步合为一步就是reduceByKey
    val wordcountRDD3: RDD[(String, Int)] = tupleRDD.reduceByKey(_ + _)
    val wordcountRDD4: RDD[(String, Int)] = tupleRDD.foldByKey(0)(_ + _)
    //8、打印结果
    println("reduceByKey:")
    wordcountRDD3.foreach(println(_))
    println("foldByKey:")
    wordcountRDD4.foreach(println(_))
    //9、用aggregateByKey来做wordcount
    val wordcountRDD5: RDD[(String, Int)] = tupleRDD.aggregateByKey(0)(_ + _, _ + _)
    //10、打印结果
    println("aggregateByKey:")
    wordcountRDD5.foreach(println(_))

    //11、关闭上下文
    sc.stop()
  }
}

• reduceBYKey , 其实是先groupByKey再reduce,如上例wordRDD.map((_, 1))之后有有(hello,1) (hello,1) (hello,1) (hello,1)，先GroupByKey，变成了 hello -> [1,1,1,1]，再reduce，对[1,1,1,1]进行聚合比如 + ， 变成了4