sparkRDD操作之ReduceByKey和GroupByKey比较
直接看代码
package cn.sparkdemo.core
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
object TestRDDFuctions {
def main(args: Array[String]): Unit = {
//1、创建上下文对象
val sparkConf: SparkConf = new SparkConf()
.setAppName(this.getClass.getSimpleName.stripSuffix("$"))
.setMaster("local[*]")
val sc = new SparkContext(sparkConf)
//2、加载数据形成RDD
val inputRDD: RDD[String] = sc.parallelize(Array(
"hello me you her",
"hello you her",
"hello her",
"hello"
))
//3、将RDD的长文本拆分成单词
val wordRDD: RDD[String] = inputRDD.flatMap(_.split("\\s+"))
//4、继续为每个单词标记为1
//tupleRDD :[(hello,1),(hello,1),(hello,1),(hello,1),(me,1),(you,1)..........]
val tupleRDD: RDD[(String, Int)] = wordRDD.map((_, 1))
//5、用groupByKey来做wordcount
//groupedRDD:[(hello,[1,1,1,1]),(me,[1]),(you,[1,1]),.......]
val groupedRDD: RDD[(String, Iterable[Int])] = tupleRDD.groupByKey()
//(x -> y) 即元组(x,y)
//wordcountRDD1:[(hello,4),(me,1),(you,2).......]
val wordcountRDD1: RDD[(String, Int)] = groupedRDD.map(x => x._1 -> x._2.sum)
//用另一个写法
val wordcountRDD2: RDD[(String, Int)] = groupedRDD.mapValues(x => x.sum)
//6、打印结果
println("groupByKey:")
wordcountRDD1.foreach(println(_))
//groupedRDD.foreach(println(_))
//7、用reduceByKey和foldByKey来做wordcount
//tupleRDD 以hello为例, 有(hello,1) (hello,1) (hello,1) (hello,1)
//reduceBYKey , 其实是先groupByKey再reduce
//先ByKey,变成了 hello -> [1,1,1,1]
//再reduce,对[1,1,1,1]进行聚合比如 _+_ , 变成了4
//最后总体变成了hello -> 4
//上面2步合为一步就是reduceByKey
val wordcountRDD3: RDD[(String, Int)] = tupleRDD.reduceByKey(_ + _)
val wordcountRDD4: RDD[(String, Int)] = tupleRDD.foldByKey(0)(_ + _)
//8、打印结果
println("reduceByKey:")
wordcountRDD3.foreach(println(_))
println("foldByKey:")
wordcountRDD4.foreach(println(_))
//9、用aggregateByKey来做wordcount
val wordcountRDD5: RDD[(String, Int)] = tupleRDD.aggregateByKey(0)(_ + _, _ + _)
//10、打印结果
println("aggregateByKey:")
wordcountRDD5.foreach(println(_))
//11、关闭上下文
sc.stop()
}
}
- reduceBYKey , 其实是先groupByKey再reduce,如上例wordRDD.map((_, 1))之后有有(hello,1) (hello,1) (hello,1) (hello,1),先GroupByKey,变成了 hello -> [1,1,1,1],再reduce,对[1,1,1,1]进行聚合比如 + , 变成了4