MapReduce编程场景之TopN 算法实现

MapReduce编程场景之TopN 算法实现

（一）需求
求出每个班参考学生成绩最高的学生的信息，班级，姓名和平均分
（二）分析
1、利用“班级和平均分”作为 key，可以将 map 阶段读取到的所有学生成绩数据按照班级和成绩排倒序，发送到 reduce
2、在 reduce 端利用 GroupingComparator 将班级相同的 kv 聚合成组， 然后取第一个即是最大值
（三）代码实现

/**
*第一步：先把分组和排序字段都综合到一个自定义对象里
*@author lv_hulk
*/

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.io.WritableComparable;
public class ClazzScore implements WritableComparable<ClazzScore>{
private String clazz;
private Double score;
public String getClazz() {
return clazz;
}
public void setClazz(String clazz) {
this.clazz = clazz;
}
public Double getScore() {
return score;
}
public void setScore(Double score) {
this.score = score;
}
public ClazzScore(String clazz, Double score) {
super();
this.clazz = clazz;
this.score = score;
}
public ClazzScore() {
super();
}
@Override
public String toString() {
return clazz + "\t" + score;
}
@Override
public void write(DataOutput out) throws IOException {
out.writeUTF(clazz);
out.writeDouble(score);
}
@Override
public void readFields(DataInput in) throws IOException {
this.clazz = in.readUTF();
this.score = in.readDouble();
}
/**
* key 排序
*/
@Override
public int compareTo(ClazzScore cs) {
int it = cs.getClazz().compareTo(this.clazz);
if(it == 0){
return (int) (cs.getScore() - this.score);
}else{
return it;
}
}
}

/**
*第二步：编写排序之后的 ClazzScore 数据传入 ReduceTask 的分组规则
*@author lv_hulk
*/
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableComparator;
public class ClazzScoreGroupComparator extends WritableComparator{
ClazzScoreGroupComparator(){
super(ClazzScore.class, true);
}
/**
* 决定输入到 reduce 的数据的分组规则
*/
@Override
public int compare(WritableComparable a, WritableComparable b) {
ClazzScore cs1 = (ClazzScore)a;
ClazzScore cs2 = (ClazzScore)b;
int it = cs1.getClazz().compareTo(cs2.getClazz());
return it;
}
}

/**
*第三步：编写 MapReduce 程序
*@author lv_hulk
*/
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
/**
* TopN 问题
*/
public class ScoreTop1MR {
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
Job job = Job.getInstance(conf);
job.setJarByClass(ScoreTop1MR.class);
job.setMapperClass(ScoreTop1MRMapper.class);
job.setReducerClass(ScoreTop1MRReducer.class);
job.setOutputKeyClass(ClazzScore.class);
job.setOutputValueClass(DoubleWritable.class);
 // 设置传入 reducer 的数据分组规则
job.setGroupingComparatorClass(ClazzScoreGroupComparator.class);
FileInputFormat.setInputPaths(job, "d:/score_all/input");
Path p = new Path("d:/score_all/output1");
FileSystem fs = FileSystem.newInstance(conf);
if(fs.exists(p)){
fs.delete(p, true);
}
FileOutputFormat.setOutputPath(job, p);
job.waitForCompletion(true);
}
static class ScoreTop1MRMapper extends Mapper<LongWritable, Text, ClazzScore, 
DoubleWritable>{
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, 
InterruptedException {
String[] splits = value.toString().split("\t");
ClazzScore cs = new ClazzScore(splits[0], Double.parseDouble(splits[2]));
context.write(cs, new DoubleWritable(Double.parseDouble(splits[2])));
}
}
static class ScoreTop1MRReducer extends Reducer<ClazzScore, DoubleWritable, ClazzScore, 
DoubleWritable>{
@Override
protected void reduce(ClazzScore cs, Iterable<DoubleWritable> scores, Context 
context) throws IOException, InterruptedException {
 // 按照规则，取每组的第一个就是 Top1
context.write(cs, scores.iterator().next());
}
}
}