1. reduceByKey
def main(args: Array[String]): Unit = {
val conf: SparkConf = new SparkConf().setMaster("local[*]").setAppName("WordCount")
val sc: SparkContext = new SparkContext(conf)
val rdd: RDD[String] = sc.makeRDD(List("hello", "world", "hive", "world", "spark", "spark", "hello"))
val rdd1: RDD[(String, Int)] = rdd.map((_, 1)).reduceByKey(_ + _)
println(rdd1.collect().mkString(","))
sc.stop()
}
2. foldByKey
def main(args: Array[String]): Unit = {
val conf: SparkConf = new SparkConf().setMaster("local[*]").setAppName("WordCount")
val sc: SparkContext = new SparkContext(conf)
val rdd: RDD[String] = sc.makeRDD(List("hello", "world", "hive", "world", "spark", "spark", "hello"))
val rdd2: RDD[(String, Int)] = rdd.map((_, 1)).foldByKey(0)(_ + _)
println(rdd2.collect().mkString(","))
sc.stop()
}
3. aggregateByKey
def main(args: Array[String]): Unit = {
val conf: SparkConf = new SparkConf().setMaster("local[*]").setAppName("WordCount")
val sc: SparkContext = new SparkContext(conf)
val rdd: RDD[String] = sc.makeRDD(List("hello", "world", "hive", "world", "spark", "spark", "hello"))
val rdd3: RDD[(String, Int)] = rdd.map((_, 1)).aggregateByKey(0)(_ + _, _ + _)
println(rdd3.collect().mkString(","