maven依赖:
<dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-core_2.10</artifactId> <version>1.6.0</version> </dependency>
public class SecondarySortBean implements Ordered<SecondarySortBean>,Serializable { private int first; private int second; public SecondarySortBean(int first, int second){ this.first = first; this.second = second; } public int compare(SecondarySortBean that) { if (this.first-that.getFirst() != 0){ return this.first-that.getFirst(); }else{ return this.second - this.getSecond(); } } public boolean $less(SecondarySortBean that) { if (this.first<that.getFirst()){ return true; }else if(this.first==that.getFirst() && this.second<that.getSecond()){ return true; } return false; } public boolean $greater(SecondarySortBean that) { if (this.first>that.getFirst()){ return true; }else if(this.first==that.getFirst() && this.second>that.getSecond()){ return true; } return false; } public boolean $less$eq(SecondarySortBean that) { if (this.$less(that)){ return true; }else if(this.first==that.getFirst() && this.second==that.getSecond()){ return true; } return false; } public boolean $greater$eq(SecondarySortBean that) { if (this.$greater(that)){ return true; }else if(this.first==that.getFirst() && this.second==that.getSecond()){ return true; } return true; } public int compareTo(SecondarySortBean that) { if (this.first-that.getFirst() != 0){ return this.first-that.getFirst(); }else{ return this.second - this.getSecond(); } } /** * Getter method for property <tt>first</tt> * * @return property value of first */ public int getFirst() { return first; } /** * Setter method for property <tt>first</tt>. * * @param first value to be assigned to property first */ public void setFirst(int first) { this.first = first; } /** * Getter method for property <tt>second</tt> * * @return property value of second */ public int getSecond() { return second; } /** * Setter method for property <tt>second</tt>. * * @param second value to be assigned to property second */ public void setSecond(int second) { this.second = second; } }
public class SecondarySortTest { public static void main(String[] args) { /** * 创建spark配置对象SparkConf,设置spark运行时配置信息, * 例如通过setMaster来设置程序要连接的集群的Master的URL,如果设置为local, * spark为本地运行 */ SparkConf conf = new SparkConf().setAppName("My first spark").setMaster("local"); /** * 创建JavaSparkContext对象 * SparkContext是spark所有功能的唯一入口, * SparkContext核心作用,初始化spark运行所需要的核心组件,同时还会负责spark程序在master的注册。 * */ JavaSparkContext sc = new JavaSparkContext(conf); /** * 根据数据来源,通过JavaSparkContext来创建RDD */ JavaRDD<String> lines = sc.textFile("E:/secondarysort.txt"); JavaPairRDD<SecondarySortBean, String> pairs = lines.mapToPair(new PairFunction<String, SecondarySortBean, String>() { public Tuple2<SecondarySortBean, String> call(String s) throws Exception { String[] split = s.split(" "); return new Tuple2<SecondarySortBean, String>( new SecondarySortBean(Integer.parseInt(split[0]), Integer.parseInt(split[1])), s); } }); JavaPairRDD<SecondarySortBean, String> sorted = pairs.sortByKey(); JavaRDD<String> map = sorted.map(new Function<Tuple2<SecondarySortBean, String>, String>() { public String call(Tuple2<SecondarySortBean, String> v1) throws Exception { return v1._2; } }); map.foreach(new VoidFunction<String>() { public void call(String s) throws Exception { System.out.println(s); } }); sc.stop(); } }