Hadoop基于WordCount的Mapper、Reducer、Combiner、Partitioner和自定义多文件输出

1    Mapper的代码

public class WorldMapper extends Mapper<Object, Text, Text, IntWritable>{

	public Text textvalue = new Text("test");
	public IntWritable intwrite = new IntWritable(1);
	
	@Override
	protected void map(Object key, Text value,Context context)
			throws IOException, InterruptedException {
		StringTokenizer stringTokenizer = new StringTokenizer(value.toString());
		while(stringTokenizer.hasMoreTokens())
		{
			textvalue.set(stringTokenizer.nextToken());
			context.write(textvalue,intwrite);
		}
	}
}

2    Reducer的代码

public class WordReduce extends Reducer<Text,IntWritable,Text, IntWritable>{

	public IntWritable value = new IntWritable();
	
	@Override
	protected void reduce(Text text, Iterable<IntWritable> values, Context context)
			throws IOException, InterruptedException {
		
		int sum = 0;
		while(values.iterator().hasNext())
		{
			sum += values.iterator().next().get();
		}
		value.set(sum + 10);
		context.write(text, value);		
	}
}

3    Combiner的代码

public class WordCombiner extends Reducer<Text, IntWritable, Text, IntWritable> {

	private IntWritable value = new IntWritable();
	@Override
	protected void reduce(Text text, Iterable<IntWritable> values, Context context)
			throws IOException, InterruptedException {
		
		int sum = 0;
		while(values.iterator().hasNext())
		{
			sum += values.iterator().next().get();
		}
		value.set(sum + 10);
		context.write(text, value);
	}
}

4    Partitioner的代码

public class WordPartioner extends Partitioner<Text, IntWritable> {

	@Override
	public int getPartition(Text text, IntWritable value, int numPartion) {
		return text.toString().hashCode() & Integer.MAX_VALUE % numPartion;
	}
}

5     自定义多文件输出

public class WordMultipleOutputFormat extends MultipleOutputFormat<Text, IntWritable>{

	@Override
	protected String generateFileNameForKeyValue(Text key, IntWritable value,
			String name) {
		char c = key.toString().toLowerCase().charAt(0);
		if(c >= 'a' && c <= 'z')
			return c+".txt";
		return "result.txt";	
	}

	@Override
	protected RecordWriter<Text, IntWritable> getBaseRecordWriter(
			FileSystem arg0, JobConf arg1, String arg2, Progressable arg3)
			throws IOException {
		// TODO Auto-generated method stub
		return null;
	}
}

6    驱动程序

public class WordCounter {
	public static void main(String[] args) throws IOException{
		
		Configuration conf = new Configuration();
		String[] otherArgs = new GenericOptionsParser(conf,args).getRemainingArgs();
		if(otherArgs.length !=2 )
		{
			System.err.println("Usage:wordcount <int><out>");
			System.exit(2);
		}
		Job job = new Job(conf,"word count");
		job.setJarByClass(WordCounter.class);
		job.setMapperClass(WorldMapper.class);
		job.setPartitionerClass(WordPartioner.class);
		job.setReducerClass(WordReduce.class);
		job.setCombinerClass(WordCombiner.class);
		job.setNumReduceTasks(4);
		job.setOutputKeyClass(Text.class);
		job.setOutputValueClass(IntWritable.class);
		//job.setOutputFormatClass(WordMultipleOutputFormat.class);
	    	FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
	    	FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
	    	try {
			System.exit(job.waitForCompletion(true) ? 0 : 1);
		} catch (InterruptedException e) {
			e.printStackTrace();
		} catch (ClassNotFoundException e) {
			e.printStackTrace();
		}
	}
}

7    运行及效果

 

 

为了显示使用了自定义的Combiner，为每个Key的值加10

问题：setOutPutFormat总是找不到自定义的多文件输出类