一个简单hadoop实例

2019独角兽企业重金招聘Python工程师标准>>>

package hadoop.test.test;

import hadoop.test.mapreduce.Test;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;

public class MaxTemperature {

	static class MaxTemperatureMapper extends
			Mapper<LongWritable, Text, Text, IntWritable> {
		private static final int MISSING = 9999;

		@Override
		protected void map(LongWritable key, Text value,
				Mapper<LongWritable, Text, Text, IntWritable>.Context context)
				throws IOException, InterruptedException {
			String line=value.toString();
			String year=line.substring(15,19);
			int airTemperature;
			if(line.charAt(87)=='+'){
				airTemperature=Integer.parseInt(line.substring(88, 92));
				
			}else{
				airTemperature=Integer.parseInt(line.substring(87, 92));
			}
			String quality=line.substring(92,93);
			if(airTemperature!=MISSING && quality.matches("[12459]")){
				context.write(new Text(year), new IntWritable(airTemperature));
			}
		}
	}
	
	static class MaxTemperatureReducer extends Reducer<Text, IntWritable, Text, IntWritable>{
		int maxValue=Integer.MIN_VALUE;

		@Override
		protected void reduce(Text key, Iterable<IntWritable> values,
				Reducer<Text, IntWritable, Text, IntWritable>.Context context)
				throws IOException, InterruptedException {

			for (IntWritable value : values) {
				maxValue=Math.max(maxValue,value.get());
				
			}
			context.write(key,new IntWritable(maxValue));
			
		}
		
	}
	
	
	public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
		
		if(args.length!=2){
			System.err.println("Usage:MaxTemperature <input path><output path>");
			System.exit(-1);
		}
		
		
		Configuration conf=new Configuration();
		
		String[] otherArgs=new GenericOptionsParser(conf, args).getRemainingArgs();
		
		
		Job job = new Job(conf, "word count");
		
		job.setJarByClass(Test.class);
		
		FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
		FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
		
		job.setMapperClass(MaxTemperatureMapper.class);
		job.setReducerClass(MaxTemperatureReducer.class);
		
		//默认输出输入类型：也是可以输出的
		job.setOutputKeyClass(Text.class);
		job.setOutputValueClass(IntWritable.class);
		
		System.exit(job.waitForCompletion(true)?0:1);
		
	}
}

 

转载于:https://my.oschina.net/yabushan/blog/704972