Spark二次排序（Java+Scala）

1.基础排序算法

sc.textFile("/data/putfile.txt").flatMap(_.split(" ")).map(word=>(word,1)).reduceByKey(_+_,1).map(pair=>(pair._2,pair._1)).sortByKey(false).map(pair=>(pair._2,pair._1)).collect

2.二次排序算法（Java实现）

import java.io.Serializable;

import scala.math.Ordered;

public class SecondarySortKey implements Ordered<SecondarySortKey>,Serializable {

    //需要二次排序的Key
    private int first;
    private int seconde;

    /**
     * 二次排序的构造方法
     * @param first
     * @param seconde
     */
    public SecondarySortKey(int first, int seconde) {
        super();
        this.first = first;
        this.seconde = seconde;
    }


    @Override
    public boolean $greater(SecondarySortKey other) {
        // TODO Auto-generated method stub
        if(this.first> other.getFirst()){
            return true;
        }else if(this.first==other.getFirst()&&this.seconde>other.seconde){
            return true;
        }
        return false;
    }
    @Override
    public boolean $greater$eq(SecondarySortKey other) {
        // TODO Auto-generated method stub
        if(this.$greater(other)){
            return true;
        }else if(this.first==other.getFirst()&&this.seconde==other.getSeconde()){
            return true;
        }
        return false;
    }
    @Override
    public boolean $less(SecondarySortKey other) {
        // TODO Auto-generated method stub
        if(this.first<other.getFirst()){
            return true;
        }else if(this.first==other.getFirst()&&this.seconde<other.getSeconde()){
            return true;
        }
        return false;
    }
    @Override
    public boolean $less$eq(SecondarySortKey other) {
        // TODO Auto-generated method stub
        if(this.$less(other)){
            return true;
        }else if(this.first==other.getFirst()&&this.seconde==other.getSeconde()){
            return true;
        }
        return false;
    }
    @Override
    public int compare(SecondarySortKey other) {
        // TODO Auto-generated method stub
        if(this.first - other.getFirst()!=0){
            return this.first-other.getFirst();
        }else{
            return this.seconde-other.getSeconde();
        }

    }
    @Override
    public int compareTo(SecondarySortKey other) {
        // TODO Auto-generated method stub
        if(this.first-other.getFirst()!=0){
            return this.first-other.getFirst();
        }else{
            return this.seconde-other.getSeconde();
        }
    }



    @Override
    public int hashCode() {
        final int prime = 31;
        int result = 1;
        result = prime * result + first;
        result = prime * result + seconde;
        return result;
    }


    @Override
    public boolean equals(Object obj) {
        if (this == obj)
            return true;
        if (obj == null)
            return false;
        if (getClass() != obj.getClass())
            return false;
        SecondarySortKey other = (SecondarySortKey) obj;
        if (first != other.first)
            return false;
        if (seconde != other.seconde)
            return false;
        return true;
    }


    public int getFirst() {
        return first;
    }


    public void setFirst(int first) {
        this.first = first;
    }


    public int getSeconde() {
        return seconde;
    }


    public void setSeconde(int seconde) {
        this.seconde = seconde;
    }


}

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.api.java.function.VoidFunction;
import scala.Tuple2;

/**
 * 二次排序：具体的实现步骤
 * 第一步：按照Ordered和Serializable接口实现自定义排序的Key
 * 第二步：将要进行二次排序的文件加载进来生成<key,value>类型的RDD
 * 第三步：使用SortByKey基于自定义的Key进行二次排序
 * 第四步：去除调排序的key，只保留排序的结果
 * 
 * @author Shuai.Zh
 *
 */
public class SecondarySortApp {

    public static void main(String[] args) {
        // TODO Auto-generated method stub

        SparkConf conf=new SparkConf();
        conf.setAppName("SecondarySort");
        conf.setMaster("local");

        JavaSparkContext sc=new JavaSparkContext(conf);
        JavaRDD<String> lines=sc.textFile("C:\\Users\\Administrator\\Desktop\\1.txt");
        JavaPairRDD<SecondarySortKey, String> pairs=lines.mapToPair(new PairFunction<String, SecondarySortKey, String>() {


            @Override
            public Tuple2<SecondarySortKey, String> call(String line) throws Exception {
                // TODO Auto-generated method stub
                String[] splited=line.split(" ");
                SecondarySortKey key=new SecondarySortKey(Integer.valueOf(splited[0]),Integer.valueOf(splited[1]));

                return new Tuple2<SecondarySortKey, String>(key, line);
            }
        });

        JavaPairRDD<SecondarySortKey, String> sorted=pairs.sortByKey();//完成二次排序
        //过滤掉排序后自定的key,保留排序的结果
        JavaRDD<String> secondarySorted=sorted.map(new Function<Tuple2<SecondarySortKey,String>, String>() {

            @Override
            public String call(Tuple2<SecondarySortKey, String> sortedCount) throws Exception {
                // TODO Auto-generated method stub
                return sortedCount._2;
            }
        });

        secondarySorted.foreach(new VoidFunction<String>() {

            @Override
            public void call(String sorted) throws Exception {
                // TODO Auto-generated method stub
                System.out.println(sorted);
            }
        });;
    }

}

二次排序Scala实现

import org.apache.spark.SparkConf

/**
  * Created by Administrator on 2017/6/1.
  */
class  SecondarySortkey(val first:Int,val second :Int) extends Ordered[SecondarySortkey] with Serializable {
    def compare(other:SecondarySortkey): Int ={
      if(this.first-other.first!=0){
       this.first-other.first
      }else{
        this.second-other.second
      }
    }
}

import org.apache.spark.{SparkConf, SparkContext}

/**
  * Created by Administrator on 2017/6/1.
  */
object SecondarySortApp {
  def main(args: Array[String]): Unit = {
    val conf=new SparkConf().setAppName("sort").setMaster("local")
    val sc=new SparkContext(conf)
    val lines=sc.textFile("C:\\Users\\Administrator\\Desktop\\1.txt")

    val pairWithSortKey=lines.map(line=> (
      new SecondarySortkey(line.split(" ")(0).toInt,line.split(" ")(1).toInt),line
      ))

    val sorted=pairWithSortKey.sortByKey(false)
    val sortedResult=sorted.map(sortedLine=>sortedLine._2)
    sortedResult.collect().foreach(println)
  }
}