package com.meng.nan.day720
import org.apache.log4j.{Level, Logger}
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.catalyst.expressions.aggregate.First
//二次排序所谓二次排序,那就是按照两(多)列进行排序,如果第一列排序结果相同,就开始考虑第二列排序。
object QuadraticRanking {
def main(args: Array[String]): Unit = {
Logger.getLogger("org.apache.spark").setLevel(Level.WARN)
Logger.getLogger("org.apache.hadoop").setLevel(Level.WARN)
Logger.getLogger("org.spark_project").setLevel(Level.WARN)
val conf=new SparkConf()
.setAppName("QuadraticRanking")
.setMaster("local[*]")
val sc=new SparkContext(conf)
esb(sc)
}
def esb(sc:SparkContext): Unit ={
val list=List(
"3 22 helo you",
"32 24 hate you",
"1 20 miss you",
"1 23 love you",
"3 23 fuck you"
)
val list1=sc.parallelize(list)
val pairs:RDD[(MySecondSort,Int)]=list1.map(line=>{
val fields=line.split("\\s+")
val mss=new MySecondSort(fields(0).toInt,fields(1).toInt)
(mss,1)
})
pairs.sortByKey(numPartitions = 1).foreach{case (mss,c)=>println(mss)}
sc.stop()
}
}
//自定义实现二次排序
class MySecondSort extends Comparable[MySecondSort] with Serializable{
var first:Int=_
var second:Int=_
def this(first:Int,second:Int){
this()
this.first=first
this.second=second
}
override def toString: String = first+"\t"+second
override def compareTo(o: MySecondSort): Int = {
var ret=this.first.compareTo((o.first))
if (ret==0 ){
ret=o.second.compareTo(this.second)
}
ret
}
}
spark二次排序
最新推荐文章于 2022-07-22 15:37:35 发布