一、前言
利用MapReduce来实现,首先按照第一列升序排列,当第一列相同时,第二列升序排列
3 3
3 2
3 1
2 2
2 1
1 1
-------------------------------------
预期结果
1 1
2 1
2 2
3 1
3 2
3 3
主要思路:
因为map输出的 <key,value>
是按照key来排序,value不能参与排序,所以这里就自定义一个key 其实现WritableComparable类,具体自定义方式见代码中的NewK2的实现部分。
二、代码
package sort;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.net.URI;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.mapreduce.lib.partition.HashPartitioner;
public class sort {
static final String INPUT_PATH = "hdfs://hadoop1:9000/input";
static final String OUT_PATH = "hdfs://hadoop1:9000/out";
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
final FileSystem fileSystem = FileSystem.get(new URI(INPUT_PATH), conf);
if(fileSystem.exists(new Path(OUT_PATH))){
fileSystem.delete(new Path(OUT_PATH), true);
}
final Job job = new Job(conf,sort.class.getSimpleName());
//指定输入目录
FileInputFormat.setInputPaths(job, new Path(INPUT_PATH));
//指定输入数据进行格式化的类
job.setInputFormatClass(TextInputFormat.class);
//指定自定义Mapper类
job.setMapperClass(MyMapper.class);
//指定Mapper输出的key,value类型
job.setMapOutputKeyClass(NewK2.class);
job.setMapOutputValueClass(LongWritable.class);
//分区
job.setPartitionerClass(HashPartitioner.class);
job.setNumReduceTasks(1);
//指定自定义的Reducer类
job.setReducerClass(MyReducer.class);
job.setOutputKeyClass(LongWritable.class);
job.setOutputValueClass(LongWritable.class);
//指定输出目录
FileOutputFormat.setOutputPath(job, new Path(OUT_PATH));
//指定输出的格式化类
job.setOutputFormatClass(TextOutputFormat.class);
//将整个作业提交给JobTracker
job.waitForCompletion(true);
}
static class MyMapper extends Mapper<LongWritable, Text, NewK2, LongWritable>{
@Override
protected void map(LongWritable key, Text v1,
Mapper<LongWritable, Text, NewK2, LongWritable>.Context context)
throws IOException, InterruptedException {
String[] splited = v1.toString().split("\t");
final long k2Long = Long.parseLong(splited[0]);
final long v2Long = Long.parseLong(splited[1]);
NewK2 k2 = new NewK2(k2Long,v2Long);
context.write(k2, new LongWritable(v2Long));
}
}
static class MyReducer extends Reducer<NewK2, LongWritable, LongWritable, LongWritable>{
@Override
protected void reduce(
NewK2 k2,
Iterable<LongWritable> v2s,
Reducer<NewK2, LongWritable, LongWritable, LongWritable>.Context context)
throws IOException, InterruptedException {
context.write(new LongWritable(k2.first), new LongWritable(k2.second));
}
}
static class NewK2 implements WritableComparable<NewK2>{
Long first;//第一列数
Long second;//第二列数
public NewK2(){}
public NewK2(Long first,Long second){
this.first = first;
this.second = second;
}
@Override
public void readFields(DataInput in) throws IOException {
this.first = in.readLong();
this.second = in.readLong();
}
@Override
public void write(DataOutput out) throws IOException {
out.writeLong(first);
out.writeLong(second);
}
/**
* key排序是会调用该方法
* 如果当第一列不同时,第一列升序,当第一列相同时,第二列升序
*/
@Override
public int compareTo(NewK2 o) {
final long minus = this.first - o.first;
if(minus != 0){
return (int)minus;
}
return (int)(this.second - o.second);
}
@Override
public int hashCode() {
return this.first.hashCode() + this.second.hashCode();
}
@Override
public boolean equals(Object obj) {
if(!(obj instanceof NewK2)){
return false;
}
NewK2 ok2 = (NewK2)obj;
return (this.first == ok2.first) && (this.second == ok2.second);
}
}
}