spark中wordcount的简单实现
package com.atguigu.spark.core.RDD2
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
/**
* @author Demigod_zhang
* @create 2020-09-05 20:49
*/
object Wc_Test {
def main(args: Array[String]): Unit = {
//定义
val sparkConf: SparkConf = new SparkConf().setAppName("groupBy").setMaster("local[*]")
val sc = new SparkContext(sparkConf)
val RDD: RDD[String] = sc.makeRDD(List("Hello", "Hello", "Spark", "Spark","scala"),2)
//groupby
// val newRDD: RDD[(String, Int)] = RDD.groupBy(word =>word).map(tup =>(tup._1,tup._2.size))
// newRDD.collect().foreach(println)
//reduceBykey
// val newRDD: RDD[(String, Int)] = RDD.map(word =>(word,1)).reduceByKey(_+_)
// newRDD.collect().foreach(println)
//groupByKey
// val newRDD: RDD[(String, Int)] = RDD.map(word =>(word,1)).groupByKey().map(tup =>(tup._1,tup._2.sum))
// newRDD.collect().foreach(println)
//aggregateByKey
// val newRDD: RDD[(String, Int)] = RDD.map(word =>(word,1)).aggregateByKey(0)(_+_,_+_)
// newRDD.collect().foreach(println)
//foldByKey
// val newRDD: RDD[(String, Int)] = RDD.map(word =>(word,1)).foldByKey(0)(_+_)
// newRDD.collect().foreach(println)
//combineBykey
// RDD.map(word =>(word,1)).combineByKey(
// num => num,
// (count: Int, num: Int) => count + num,
// (count1: Int, count2: Int) => count1 + count2
// ).collect().foreach(println)
//countByKey
// val newRDD: collection.Map[String, Long] = RDD.map(word =>(word,1)).countByKey()
// newRDD.foreach(println)
//countByValue
// val newRDD: collection.Map[String, Long] = RDD.map(word =>word).countByValue()
// newRDD.foreach(println)
//cogroup
// val rdd2 = sc.makeRDD(List(("a", 0)))
// val newRDD: RDD[(String, Int)] = RDD.map((_, 1)).cogroup(rdd2).map {
// case (word, (iter1, iter2)) => {
// (word, iter1.size + iter2.size)
// }
// }.filter(tup => tup._1 != "a")
// newRDD
// newRDD.collect().foreach(println)
//mapvalues
// val newRDD: RDD[(String, Int)] = RDD.map((_,1)).groupBy(word =>word).mapValues(_.size).map{case ((word, num), count) => (word, count)}
// newRDD.collect().foreach(println)
}
}