MapReduce-TopN找最小数

最新推荐文章于 2024-08-30 16:00:21 发布

It_sucks

最新推荐文章于 2024-08-30 16:00:21 发布

阅读量107

点赞数

文章标签： hadoop 大数据 mapreduce

本文链接：https://blog.csdn.net/weixin_71132847/article/details/130800770

版权

该代码示例展示了如何使用HadoopMapReduce进行TopN数据处理。TopNMapper类将输入数据存储在TreeMap中并保留最大的N个数，超过10个时移除最小值。TopNReducer则接收Mapper的输出，维护一个反向排序的TreeMap，确保始终保留最大的10个数。TopNDriver配置并提交MapReduce作业。

摘要由CSDN通过智能技术生成

TopNMapper

package mr.topN;

import java.util.TreeMap;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

/*
* TreeMap save key-value
*/
public class TopNMapper extends Mapper<LongWritable, Text, NullWritable, IntWritable> {
private TreeMap<Integer,String>repToRecordMap=new TreeMap<Integer,String>();
@Override
public void map(LongWritable key,Text vaule,Context context){
String line=vaule.toString();
String[] nums=line.split(" ");
for (String num:nums){
/*
* read data write TreeMap，>5 move minimum number
*/
repToRecordMap.put(Integer.parseInt(num),",");
if (repToRecordMap.size()>10){
repToRecordMap.remove(repToRecordMap.lastKey());
}
}
}
/*
* write cleanup function
* read all data , output reduce
*/
@Override
protected void cleanup(Context context){
for (Integer i:repToRecordMap.keySet()){
try{
context.write(NullWritable.get(),new IntWritable(i));
}catch (Exception e){
e.printStackTrace();
}
}
}
}

TopNReducer

package mr.topN;

import java.io.IOException;
import java.util.Comparator;
import java.util.TreeMap;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.Reducer;

public class TopNReducer extends Reducer<NullWritable,IntWritable,NullWritable,IntWritable> {
//create TreeMap
private TreeMap<Integer,String>repToRecordMap=new TreeMap<Integer,String>(new Comparator<Integer>() {

//return plural ：a<b
//return 0 ：a=b
//return positive number ：a>b
public int compare(Integer a, Integer b) {
return b-a;
}
});
public void reduce(NullWritable key,
Iterable<IntWritable>vaules,Context context)throws IOException,InterruptedException{
for (IntWritable value:vaules){
repToRecordMap.put(value.get(),"");
if (repToRecordMap.size()>10){
repToRecordMap.remove(repToRecordMap.firstKey());
}

}
for (Integer i:repToRecordMap.keySet()){
context.write(NullWritable.get(),new IntWritable(i));
}
}
}

TopNDriver

package mr.topN;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.Job;

public class TopNDriver {
public static void main(String[] args)throws Exception{
Configuration configuration=new Configuration();
Job job=Job.getInstance(configuration);
job.setJarByClass(TopNDriver.class);
job.setMapperClass(TopNMapper.class);
job.setReducerClass(TopNReducer.class);
job.setNumReduceTasks(1);
//map output key
job.setMapOutputKeyClass(NullWritable.class);
//map output value
job.setMapOutputValueClass(IntWritable.class);
//reduce output key
job.setOutputKeyClass(NullWritable.class);
//reduce output value
job.setOutputValueClass(IntWritable.class);

FileInputFormat.setInputPaths(job,new Path("D:\\mr\\intput"));
FileOutputFormat.setOutputPath(job,new Path("D:\\mr\\output"));
boolean res=job.waitForCompletion(true);
System.exit(res?0:1);
}
}