spark之Action操作算子

package com.meng.nan.day717

import org.apache.log4j.{Level, Logger}
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.rdd.RDD

//常见的行动算子
// foreach(循环遍历),collect（转换为Array数组）,reduce（与reduceByKey主作用相同但是是action算子）
// ,saveAshadoopFile/saveAsHadoopAPIFile（都是存储到hdfs上，只是使用的API不同）、saveAsTextFile（存储操本地）等
object ActionClass {
  def main(args: Array[String]): Unit = {
    Logger.getLogger("org.apache.spark").setLevel(Level.WARN)
    Logger.getLogger("org.apache.hadoop").setLevel(Level.WARN)
    Logger.getLogger("org.spark_project").setLevel(Level.WARN)
val conf=new SparkConf()
      .setAppName("ActionClass")
      .setMaster("local[*]")
val  sc=new SparkContext(conf)
//tft2OPs(sc)
    cbk2Ops(sc)
  }
  // take(n)/frist/takeOrdered算子
  def tft2OPs(sc:SparkContext): Unit ={
    val list=List(12,4,2,32,12,21)
    val listRDD:RDD[Int]=sc.parallelize(list)
    listRDD.take(3).foreach(println)
    println("*******************")
    listRDD.top(3).foreach(println)//默认倒序排序
    println("*******************")
    println(listRDD.first())
    println("*******************")
    listRDD.takeOrdered(3).foreach(println)//和take相反的规则，正序排序



  }
  def cbk2Ops(sc:SparkContext): Unit ={
    val list=List(
      "hello you hello me",
      "hello you shit me",
      "hello you oh shit",
      "me you hello me"
    )
    val listRDD:RDD[String]=sc.parallelize(list)
    val pairsRDD:RDD[(String,Int)]=listRDD.flatMap(_.split("\\s+")).map((_,1))
    val redRDD=pairsRDD.reduceByKey(_+_)
    redRDD.foreach(println)
    println("==============")
    redRDD.take(3).foreach(println)
    println("*******************")
    redRDD.top(3).foreach(println)//默认倒序排序
    println("*******************")
    println(redRDD.first())
    println("*******************")
    redRDD.takeOrdered(3).foreach(println)//和take相反的规则，正序排序
   //自定义排序规则
    implicit val order = new Ordering[(String, Int)](){
      override def compare(x: (String, Int), y: (String, Int)) = {
        var ret = y._2.compareTo(x._2)
        if(ret == 0) {
          ret = y._1.compareTo(x._1)
        }
        ret
      }
    }
    redRDD.takeOrdered(5).foreach(println)
    println("aaaaaaaaaaaaaaaaaa")
    val cbkRDD=pairsRDD.countByKey()
    for ((k,v)<- cbkRDD){
      println(s"${k}---${v}")
    }
  }
}