Hadoop_MapReduce_求最大值和最小值

Hadoop_MapReduce_求最大值和最小值

1. 原始数据：D:data ums.txt
   需求：找出最大和最小的数字

   3
   4
   1
   10
   15
   22
   8
   hadoop
   5K
   62
   1

2. MaxMin

   package com.blu.maxmin;

   import java.io.DataInput;
   import java.io.DataOutput;
   import java.io.IOException;

   import org.apache.hadoop.io.WritableComparable;

   public class MaxMin implements WritableComparable{

   private int max;
   private int min;
   private int total;
   
   public void readFields(DataInput in) throws IOException {
   	max = in.readInt();
   	min = in.readInt();
   	total = in.readInt();
   }
   
   public void write(DataOutput out) throws IOException {
   	out.writeInt(max);
   	out.writeInt(min);
   	out.writeInt(total);
   }
   
   public int compareTo(MaxMin o) {
   	return 0;
   }
   
   public int getMax() {
   	return max;
   }
   
   public void setMax(int max) {
   	this.max = max;
   }
   
   public int getMin() {
   	return min;
   }
   
   public void setMin(int min) {
   	this.min = min;
   }
   
   public int getTotal() {
   	return total;
   }
   
   public void setTotal(int total) {
   	this.total = total;
   }
   
   @Override
   public String toString() {
   	return "MaxMin [max=" + max + ", min=" + min + ", total=" + total + "]";
   }
   
   public MaxMin() {
   	super();
   }
   
   public MaxMin(int max, int min, int total) {
   	super();
   	this.max = max;
   	this.min = min;
   	this.total = total;
   }
   

   }

3. MaxMinMapper

   package com.blu.maxmin;

   import java.io.IOException;

   import org.apache.hadoop.io.LongWritable;
   import org.apache.hadoop.io.NullWritable;
   import org.apache.hadoop.io.Text;
   import org.apache.hadoop.mapreduce.Mapper;

   public class MaxMinMapper extends Mapper<LongWritable, Text, MaxMin, NullWritable>{

   private int max;
   private int min;
   private int total = 0;
   
   @Override
   protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, MaxMin, NullWritable>.Context context)
   		throws IOException, InterruptedException {
   	String val = value.toString();
   	try {
   		int current = Integer.parseInt(val);
   		if(total == 0) {
   			min = current;
   			max = current;
   		}
   		if(current>max) {
   			max = current;
   		}
   		if(current<min) {
   			min = current;
   		}
   	} catch (Exception e) {
   		e.printStackTrace();
   		return;
   	}
   	total++;
   }
   
   /**
    * cheanup方法会在所有的map方法执行完成后执行，只会执行一次。
    */
   @Override
   protected void cleanup(Mapper<LongWritable, Text, MaxMin, NullWritable>.Context context)
   		throws IOException, InterruptedException {
   	MaxMin mm = new MaxMin(max,min,total);
   	context.write(mm, NullWritable.get());
   }
   

   }

4. MaxMinReduce

   package com.blu.maxmin;

   import java.io.IOException;

   import org.apache.hadoop.io.NullWritable;
   import org.apache.hadoop.mapreduce.Reducer;

   public class MaxMinReduce extends Reducer<MaxMin, NullWritable, MaxMin, NullWritable>{
   @Override
   protected void reduce(MaxMin key, Iterable value,
   Reducer<MaxMin, NullWritable, MaxMin, NullWritable>.Context context) throws IOException, InterruptedException {
   context.write(key, NullWritable.get());
   }
   }

5. MaxMinJob

   package com.blu.maxmin;

   import org.apache.hadoop.fs.Path;
   import org.apache.hadoop.io.NullWritable;
   import org.apache.hadoop.mapreduce.Job;
   import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
   import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

   public class MaxMinJob {
   public static void main(String[] args) throws Exception {
   Job job = Job.getInstance();
   job.setJarByClass(MaxMinJob.class);
   job.setMapperClass(MaxMinMapper.class);
   job.setReducerClass(MaxMinReduce.class);
   job.setMapOutputKeyClass(MaxMin.class);
   job.setMapOutputValueClass(NullWritable.class);
   job.setOutputKeyClass(MaxMin.class);
   job.setOutputValueClass(NullWritable.class);
   FileInputFormat.addInputPath(job, new Path(args[0]));
   FileOutputFormat.setOutputPath(job, new Path(args[1]));
   boolean flag = job.waitForCompletion(true);
   System.exit(flag ? 0 : 1);
   }
   }

6. 运行参数：

   D:data
   ums.txt D:dataoutput

7. 运行结果：

   MaxMin [max=62, min=1, total=9]