spark java 二次排序_spark java API 实现二次排序

package com.spark.sort;

import java.io.Serializable;

import scala.math.Ordered;

public class SecondSortKey implements Serializable, Ordered {

/**

* serialVersionUID

*/

private static final long serialVersionUID = -2749925310062789494L;

private String first;

private long second;

public SecondSortKey(String first, long second) {

super();

this.first = first;

this.second = second;

}

public String getFirst() {

return first;

}

public void setFirst(String first) {

this.first = first;

}

public long getSecond() {

return second;

}

public void setSecond(long second) {

this.second = second;

}

@Override

public int hashCode() {

final int prime = 31;

int result = 1;

result = prime * result + ((first == null) ? 0 : first.hashCode());

result = prime * result + (int) (second ^ (second >>> 32));

return result;

}

@Override

public boolean equals(Object obj) {

if (this == obj)

return true;

if (obj == null)

return false;

if (getClass() != obj.getClass())

return false;

SecondSortKey other = (SecondSortKey) obj;

if (first == null) {

if (other.first != null)

return false;

} else if (!first.equals(other.first))

return false;

if (second != other.second)

return false;

return true;

}

@Override

public boolean $greater(SecondSortKey that) {

if (this.first.compareTo(that.getFirst()) > 0) {

return true;

} else if (this.first.equals(that.getFirst()) && this.second > that.getSecond()) {

return true;

}

return false;

}

@Override

public boolean $greater$eq(SecondSortKey that) {

if (this.$greater(that)) {

return true;

}else if(this.first.equals(that.getFirst()) && this.second == that.getSecond()){

return true;

}

return false;

}

@Override

public boolean $less(SecondSortKey that) {

if (this.first.compareTo(that.getFirst()) < 0) {

return true;

} else if (this.first.equals(that.getFirst()) && this.second < that.getSecond()) {

return true;

}

return false;

}

@Override

public boolean $less$eq(SecondSortKey that) {

if (this.$less(that)) {

return true;

}else if(this.first.equals(that.getFirst()) && this.second == that.getSecond()){

return true;

}

return false;

}

@Override

public int compare(SecondSortKey that) {

if (this.first.compareTo(that.getFirst()) != 0) {

return this.first.compareTo(that.getFirst());

} else {

return (int) (this.second - that.getSecond());

}

}

@Override

public int compareTo(SecondSortKey that) {

if (this.first.compareTo(that.getFirst()) != 0) {

return this.first.compareTo(that.getFirst());

} else {

return (int) (this.second - that.getSecond());

}

}

}

1

package com.spark.sort;

2

3 import org.apache.spark.SparkConf;

4 import org.apache.spark.api.java.JavaPairRDD;

5 import org.apache.spark.api.java.JavaRDD;

6 import org.apache.spark.api.java.JavaSparkContext;

7 import org.apache.spark.api.java.function.Function;

8 import org.apache.spark.api.java.function.PairFunction;

9

10 import scala.Tuple2;

11

12 public class SecondSort {

13

14 public static void main(String[] args) {

15 SparkConf sparkConf = new SparkConf().setAppName("secondsort").setMaster("local");

16 JavaSparkContext jsc = new JavaSparkContext(sparkConf);

17 JavaRDD textFileRDD = jsc.textFile("D:\\test\\input\\sort");

18 JavaPairRDD pairRDD = textFileRDD.mapToPair(new PairFunction() {

19 @Override

20 public Tuple2 call(String t) throws Exception {

21 String[] split = t.split("\t");

22 String first = split[0];

23 Long second = Long.valueOf(split[1]);

24 SecondSortKey ssk = new SecondSortKey(first, second);

25 return new Tuple2(ssk, t);

26 }

27 });

28

29 //排序

30 JavaPairRDD sortByKeyRDD =pairRDD.sortByKey();

31

32 //过滤自定义的key

33 JavaRDD mapRDD = sortByKeyRDD.map(new Function, String>() {

34

35 @Override

36 public String call(Tuple2 v1) throws Exception {

37

38 return v1._2;

39 }

40 });

41

42 mapRDD.saveAsTextFile("D:\\test\\output\\sort");

43

44 jsc.close();

45 }

46

47

48 }

源数据：

a 12

a2

b26

c85

ab32

ab23

ac12

b85

a36

b69

c25

排序之后：

a 2

a12

a36

ab23

ab32

ac12

b26

b69

b85

c25

c85