MapReduce API

ProvincPar

ProvincPar
package a.b.c;

import java.io.IOException;
import java.util.HashMap;
import java.util.Map;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Partitioner;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;


class ProParMapper extends Mapper<LongWritable, Text, Text, IntWritable>{
	 @Override
	protected void map(LongWritable key, Text value,Context context)
			throws IOException, InterruptedException {
		 String [] provString=value.toString().trim().split(",");
		 int provCode=Integer.parseInt(provString[0]);
		 String provName=provString[1];
		 
		 context.write(new Text(provName), new IntWritable(provCode));
	}
		
}

class ProParPartition extends Partitioner<Text, IntWritable>{
	
	private Map<String, Integer> getProMap(){
		String[] provinceString={"湖南省","湖北省","北京市","上海市"};
		Map<String,Integer> map = new HashMap<String,Integer>();
		for (int i=0;i<(provinceString.length);i++){
			map.put(provinceString[i],i);
		}
		return map;
	}
	//reduce分组数
	@Override
	public int getPartition(Text key, IntWritable values, int numPartitions) {
		Map<String, Integer> map = getProMap();
		if (map.containsKey(key.toString().trim())) {
			System.out.println(map.get(key.toString()));
			System.out.println(map.size());
			return map.get(key.toString().trim());
		} else {
			return map.size() + 1;
		}
	}
}


class ProParReducer extends Reducer<Text, IntWritable, Text, IntWritable>{
	@Override
	protected void reduce(Text key, Iterable<IntWritable> value,Context context)
			throws IOException, InterruptedException {
		int sum=0;
		for (IntWritable each:value){
			sum+=each.get();
		}
		context.write(key,new IntWritable(sum));
	}
	
	
}


public class ProvincPar {
	public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
		Configuration configuration=new Configuration();
		Job job=Job.getInstance(configuration);
		job.setJarByClass(ProvincPar.class);
		
		job.setMapperClass(ProParMapper.class);
		job.setMapOutputKeyClass(Text.class);
		job.setMapOutputValueClass(IntWritable.class);
		
		job.setPartitionerClass(ProParPartition.class);
		
		job.setReducerClass(ProParReducer.class);
		job.setOutputKeyClass(Text.class);
		job.setOutputValueClass(IntWritable.class);
		job.setNumReduceTasks(6);
		
		Path path = new Path(args [1]);
		FileSystem fileSystem = path.getFileSystem(configuration);
		if (fileSystem.exists(path)) {
			fileSystem.delete((path),true);
		}

		FileInputFormat.addInputPath(job, new Path(args[0]));
		FileOutputFormat.setOutputPath(job, new Path(args[1]));
		
		
		System.exit(job.waitForCompletion(true) ? 0 : 1);
		
		
	}
}