Hadoop_MapReduce_求最大值和最小值
-
原始数据:D:data ums.txt
需求:找出最大和最小的数字3
4
1
10
15
22
8
hadoop
5K
62
1 -
MaxMin
package com.blu.maxmin;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;import org.apache.hadoop.io.WritableComparable;
public class MaxMin implements WritableComparable{
private int max; private int min; private int total; public void readFields(DataInput in) throws IOException { max = in.readInt(); min = in.readInt(); total = in.readInt(); } public void write(DataOutput out) throws IOException { out.writeInt(max); out.writeInt(min); out.writeInt(total); } public int compareTo(MaxMin o) { return 0; } public int getMax() { return max; } public void setMax(int max) { this.max = max; } public int getMin() { return min; } public void setMin(int min) { this.min = min; } public int getTotal() { return total; } public void setTotal(int total) { this.total = total; } @Override public String toString() { return "MaxMin [max=" + max + ", min=" + min + ", total=" + total + "]"; } public MaxMin() { super(); } public MaxMin(int max, int min, int total) { super(); this.max = max; this.min = min; this.total = total; }
}
-
MaxMinMapper
package com.blu.maxmin;
import java.io.IOException;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;public class MaxMinMapper extends Mapper<LongWritable, Text, MaxMin, NullWritable>{
private int max; private int min; private int total = 0; @Override protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, MaxMin, NullWritable>.Context context) throws IOException, InterruptedException { String val = value.toString(); try { int current = Integer.parseInt(val); if(total == 0) { min = current; max = current; } if(current>max) { max = current; } if(current<min) { min = current; } } catch (Exception e) { e.printStackTrace(); return; } total++; } /** * cheanup方法会在所有的map方法执行完成后执行,只会执行一次。 */ @Override protected void cleanup(Mapper<LongWritable, Text, MaxMin, NullWritable>.Context context) throws IOException, InterruptedException { MaxMin mm = new MaxMin(max,min,total); context.write(mm, NullWritable.get()); }
}
-
MaxMinReduce
package com.blu.maxmin;
import java.io.IOException;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.Reducer;public class MaxMinReduce extends Reducer<MaxMin, NullWritable, MaxMin, NullWritable>{
@Override
protected void reduce(MaxMin key, Iterable value,
Reducer<MaxMin, NullWritable, MaxMin, NullWritable>.Context context) throws IOException, InterruptedException {
context.write(key, NullWritable.get());
}
} -
MaxMinJob
package com.blu.maxmin;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;public class MaxMinJob {
public static void main(String[] args) throws Exception {
Job job = Job.getInstance();
job.setJarByClass(MaxMinJob.class);
job.setMapperClass(MaxMinMapper.class);
job.setReducerClass(MaxMinReduce.class);
job.setMapOutputKeyClass(MaxMin.class);
job.setMapOutputValueClass(NullWritable.class);
job.setOutputKeyClass(MaxMin.class);
job.setOutputValueClass(NullWritable.class);
FileInputFormat.addInputPath(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
boolean flag = job.waitForCompletion(true);
System.exit(flag ? 0 : 1);
}
} -
运行参数:
D:data
ums.txt D:dataoutput -
运行结果:
MaxMin [max=62, min=1, total=9]