ThirdSort_scala

最新推荐文章于 2019-05-27 11:35:15 发布

Silver_Sword

最新推荐文章于 2019-05-27 11:35:15 发布

阅读量175

点赞数

本文链接：https://blog.csdn.net/Silver_Sword/article/details/80592434

版权

本文介绍了一个使用Apache Spark进行三次排序的具体实现案例。通过定义自定义排序实体类并利用Spark RDD API，实现了基于元组的三次排序操作。文章展示了从创建Spark配置到执行排序并收集结果的完整过程。

摘要由CSDN通过智能技术生成

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

/**
  * Created by MC on 2018/6/6.
  * 三次排序
  */
object ThirdSort {
  def myThirdSort(sc : SparkContext): String ={

    val array : Array[Tuple3[Int,Int,Int]]= Array((1,1,5),(1,1,2),(1,2,1),(1,2,4),(1,2,5),
      (1,2,1),(1,1,5),(2,1,5),(2,1,2),(2,2,1),(2,2,4),(3,2,5),
  (3,2,1),(3,1,5))
    val rdd1 : RDD[Tuple3[Int,Int,Int]]= sc.parallelize(array)
    val map : RDD[(ThirdSortEntity,(Int,Int,Int))]= rdd1.map(x=>(new ThirdSortEntity(x._1,x._2,x._3),x))
    val rdd2 = map.sortByKey()
    rdd2.collect().toList.toString()

  }

  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setMaster("local").setAppName("ThirdSort")
    conf.set("spark.serializer","org.apache.spark.serializer.KryoSerializer")
    conf.set("spark.local.dir","D:/tmp")
    val sc = new SparkContext(conf)
    var result : String = ""
    result = myThirdSort(sc)
    println(result)
    sc.stop()
  }
}

class ThirdSortEntity(val num1:Int, val num2:Int,val num3:Int) extends Ordered[ThirdSortEntity] with Serializable{
 //定义排序规则
  override def compare(that: ThirdSortEntity): Int = {
  var result = this.num1-that.num1
  if(result == 0)result = that.num2-this.num2
  if(result == 0)result = this.num3-that.num3
  result
  }
}