这个问题最主要的俩个问题:
1.去重(在MAP端去重)
2.数组打擂
现在说明一下打擂
static class MyMapper extends Mapper<LongWritable, Text, LongWritable, Text>{
//定义一个擂主
final int top = 100;
final long[] arr = new long[top];
protected void map(LongWritable key, Text value, org.apache.hadoop.mapreduce.Mapper<LongWritable,Text,LongWritable,Text>.Context context) throws java.io.IOException ,InterruptedException {
// add(key.get());
String[] str = value.toString().split("\t");
add(Long.parseLong(str[0].toString()));
};
private void add(long key) {
//去掉map重复项
for(long l : arr){
if(l==key){
return;
}
}
if(key>arr[0]){
int i =0;
arr[0] = key;
for (;i<99&&key>arr[i+1]; i++) {
arr[i] =arr[i+1];
}
arr[i] = key;
}
}
/**
* 当所有的reducer任务执行完后,调用
*/
protected void cleanup(org.apache.hadoop.mapreduce.Mapper<LongWritable,Text,LongWritable,Text>.Context context) throws java.io.IOException ,InterruptedException {
for (int i =0;i<100;i++){
context.write(new LongWritable(arr[i]), new Text(arr[i]+""));
}
}
}
static class MyReduce extends Reducer<LongWritable, Text, LongWritable, Text>{
protected void reduce(LongWritable key, java.lang.Iterable<Text> value, org.apache.hadoop.mapreduce.Reducer<LongWritable,Text,LongWritable,Text>.Context arg2) throws java.io.IOException ,InterruptedException {
add(key.get());
};
//定义一个擂主
final int top = 100;
final long[] arr = new long[top];
private void add(long key) {
int i =0;
if(key>arr[0]){
arr[0] = key;
for ( ; i<99&&key>arr[i+1]; i++) {
arr[i] =arr[i+1];
}
arr[i+1] = key;
}
}
/**
* 当所有的map任务执行完后,调用
*/
protected void cleanup(org.apache.hadoop.mapreduce.Reducer<LongWritable,Text,LongWritable,Text>.Context context) throws java.io.IOException ,InterruptedException {
for (int i =0;i<100;i++){
context.write(new LongWritable(arr[i]), new Text(arr[i]+""));
}
}
}
实现效果,降序: