mapreduce程序reduce输出控制

1，在hadoop中，reduce支持多个输出,输出的文件名也是可控的，就是继承MultipleTextOutputFormat类，重写generateFileNameForKey方法

public class LzoHandleLogMr extends Configured implements Tool {

	 static class LzoHandleLogMapper extends MapReduceBase implements Mapper<LongWritable, Text, Text, Text> {
       
	  
		public void map(LongWritable key, Text value, OutputCollector<Text, Text> output, Reporter reporter)
				throws IOException {
	    	try {
	    	    String[] sp = value.toString().split(",");
	    		output.collect(new Text(sp[0]), value);
	    	}catch (Exception e) {
			   e.printStackTrace();
		    }    	
		}


	}
	static class LzoHandleLogReducer  extends MapReduceBase implements Reducer<Text, Text, Text, NullWritable> {
        


		@Override
		public void reduce(Text key, Iterator<Text> values,
				OutputCollector<Text, NullWritable> output, Reporter reporter)
				throws IOException {
			while (values.hasNext()) {
		   		  output.collect(values.next(), NullWritable.get());   
		   	   }
			
		}	
	}
	
	public static class LogNameMultipleTextOutputFormat extends MultipleTextOutputFormat<Text, NullWritable> 
	   {


		@Override
		protected String generateFileNameForKeyValue(Text key,
				NullWritable value, String name) {
			String sp[] = key.toString().split(",");
			String filename = sp[0];
			if(sp[0].contains(".")) filename="000000000000";
			return filename;
		}
		
	}
    


	@Override
	public int run(String[] args) throws Exception {
		 
		    JobConf jobconf = new JobConf(LzoHandleLogMr.class);
		    jobconf.setMapperClass(LzoHandleLogMapper.class);
		    jobconf.setReducerClass(LzoHandleLogReducer.class);
		    jobconf.setOutputFormat(LogNameMultipleTextOutputFormat.class);
		    jobconf.setOutputKeyClass(Text.class);
		    jobconf.setNumReduceTasks(12);
		    
		    
		 FileInputFormat.setInputPaths(jobconf,new Path(args[0]));
	    	FileOutputFormat.setOutputPath(jobconf,new Path(args[1]));
	    	FileOutputFormat.setCompressOutput(jobconf, true);
	    	FileOutputFormat.setOutputCompressorClass(jobconf, LzopCodec.class);  
	    	
	    	JobClient.runJob(jobconf);
	      return 0;
			
	}
}

在新版本的hadoopAPI是通过Job类来设置各种参数的，但是我调用 Job.setOutputFormatClass()来使用MultipleTextOutputFormat的时候，竟然报错，原因是必须继承子org.apache.hadoop.mapreduce.OutputFormat。0.20.2比较致命的其中一个bug， 升级到0.21能解决

2, 如果同一行数据，需要同时输出至多个文件的话，我们可以使用MultipleOutputs类：

1. publicclassMultiFileextendsConfiguredimplementsTool{
2. publicstaticclassMapClassextendsMapReduceBase
3. implementsMapper<LongWritable,Text,NullWritable,Text>{
4. privateMultipleOutputsmos;
6. privateOutputCollector<NullWritable,Text>collector;
7. publicvoidconfigure(JobConfconf){
8. mos=newMultipleOutputs(conf);
9. }
11. publicvoidmap(LongWritablekey,Textvalue,
12. OutputCollector<NullWritable,Text>output,
13. Reporterreporter)throwsIOException{
14. String[]arr=value.toString().split(",",-1);
15. Stringchrono=arr[0]+","+arr[1]+","+arr[2];
16. Stringgeo=arr[0]+","+arr[4]+","+arr[5];
17. collector=mos.getCollector("chrono",reporter);
18. collector.collect(NullWritable.get(),newText(chrono));
19. collector=mos.getCollector("geo",reporter);
20. collector.collect(NullWritable.get(),newText(geo));
21. }
23. publicvoidclose()throwsIOException{
24. mos.close();
25. }
26. }
28. publicintrun(String[]args)throwsException{
29. Configurationconf=getConf();
30. JobConfjob=newJobConf(conf,MultiFile.class);
31. Pathin=newPath(args[0]);
32. Pathout=newPath(args[1]);
33. FileInputFormat.setInputPaths(job,in);
34. FileOutputFormat.setOutputPath(job,out);
35. job.setJobName("MultiFile");
36. job.setMapperClass(MapClass.class);
37. job.setInputFormat(TextInputFormat.class);
38. job.setOutputKeyClass(NullWritable.class);
39. job.setOutputValueClass(Text.class);
40. job.setNumReduceTasks(0);
41. MultipleOutputs.addNamedOutput(job,
42. "chrono",
43. TextOutputFormat.class,
44. NullWritable.class,
45. Text.class);
46. MultipleOutputs.addNamedOutput(job,
47. "geo",
48. TextOutputFormat.class,
49. NullWritable.class,
50. Text.class);
51. JobClient.runJob(job);
52. return0;
53. }
54. }

这个类维护了一个<name, OutputCollector>的map。我们可以在job配置里添加collector，然后在reduce方法中，取得对应的collector并调用collector.write即可。