WordCount

最新推荐文章于 2024-04-09 15:24:13 发布

焚香余音

最新推荐文章于 2024-04-09 15:24:13 发布

阅读量165

点赞数

分类专栏： spark 文章标签： wordcount

本文链接：https://blog.csdn.net/SuperProgram00/article/details/85990528

版权

spark 专栏收录该内容

5 篇文章 0 订阅

订阅专栏

1. def wc1(): Unit = {

    val conf = new SparkConf().setAppName("wc").setMaster("local[2]");

    val sc = new SparkContext(conf)

    val wc = sc.textFile("E://java//HelloSpark//src//word.txt").flatMap(_.split(" ")).

      map((, 1)).reduceByKey((_ + _)).foreach(println)

    sc.stop()

  }
val words = Array("one", "two", "two", "three", "three", "three")  

val wordPairsRDD = sc.parallelize(words).map(word => (word, 1))  

val wordCountsWithReduce = wordPairsRDD.reduceByKey(_ + _)  
val wordCountsWithGroup = wordPairsRDD.groupByKey().map(t => (t.1, t.2.sum))

def wc2(): Unit = {

    val conf = new SparkConf().setAppName("wc").setMaster("local[2]");

    val sc = new SparkContext(conf)

    val wc = sc.textFile("E://java//HelloSpark//src//word.txt").flatMap(.split(" ")).map((, 1)).groupBy(.1).mapValues(.foldLeft(0)( + .2))
   wc.foreach(println)

    sc.stop()

  def wc3(): Unit = {

    val conf = new SparkConf().setAppName("wc").setMaster("local[2]");

    val sc = new SparkContext(conf)

    val wc = sc.textFile("E://java//HelloSpark//src//word.txt").

      flatMap(.split(" ")).map((, 1)).foldByKey(0)(_ + _)

    wc.foreach(println)

    sc.stop()

  }

def wc4(): Unit = {

    val conf = new SparkConf().setAppName("wc").setMaster("local[2]");

    val sc = new SparkContext(conf)

    val wc = sc.textFile("E://java//HelloSpark//src//word.txt").

      flatMap(.split(" ")).map((, 1))

      .combineByKey(x => x, (a: Int, b: Int) => a + b, (m: Int, n: Int) => m + n)

    wc.foreach(println)

    sc.stop()

  }

def wc5(): Unit = {

    val conf = new SparkConf().setAppName("wc").setMaster("local[2]");

    val sc = new SparkContext(conf)

    val wc = sc.textFile("E://java//HelloSpark//src//word.txt").

      flatMap(.split(" ")).map((, 1)).groupBy(.1)

      .map(x => (x.1, x.2.size))

    wc.foreach(println)

    sc.stop()

  }

def wc6(): Unit ={

    val conf = new SparkConf().setAppName("wc").setMaster("local[2]");

    val sc = new SparkContext(conf)

    val wc = sc.textFile("E://java//HelloSpark//src//word.txt")

      .flatMap(.split(" ")).map((,1)).aggregateByKey(0)(+,+)

     wc.foreach(println)

    sc.stop()

  }

def  wc7(): Unit ={

    val conf = new SparkConf().setAppName("wc").setMaster("local[2]");

    val sc = new SparkContext(conf)

    val wc = sc.textFile("E://java//HelloSpark//src//word.txt")

      .flatMap(.split(" ")).map((,1)).groupBy(.1).mapValues(_.size)//x=>x.size

     .foreach(println)

    sc.stop()

  }



  def wc8(): Unit ={

    val conf = new SparkConf().setAppName("wc").setMaster("local[2]");

    val sc = new SparkContext(conf)

    val wc = sc.textFile("E://java//HelloSpark//src//word.txt")

      .flatMap(.split(" ")).map((,1)).groupBy(.1).mapValues(.map((.2)).reduce(+_))

     .foreach(println)

    sc.stop()

  }

def wc9(): Unit ={

    val conf = new SparkConf().setAppName("wc").setMaster("local[2]");

    val sc = new SparkContext(conf)

    val wc = sc.textFile("E://java//HelloSpark//src//word.txt")

      .flatMap(.split(" ")).map((,1)).countByKey()

     .foreach(println)

    sc.stop()

  }

 def wc10(): Unit ={

    val conf= new SparkConf().setMaster("local[2]").setAppName("wc")

    val sc = new SparkContext(conf)

    val wc = sc.textFile("E://java//HelloSpark//src//word.txt").flatMap(_.split(" ")).

    countByValue().foreach(println)

  }