hadoop划分为多个输出文件

现在我们见到的MapReduce作业的输出都是一组文件，那如果我想输出多组文件怎么办，比如说我想统计每个国家的专利情况，想以国家名作为文件名来输出。我们可以使用MultipleOutputFormat，它内部有一个方法generateFileNameForKeyValue，只要Override他，就可以根据自己的需要划分文件。他还有一些子类，像MultipleTextOutputFormat，MultipleSequenceFileOutputFormat

import java.io.IOException;
import java.util.Iterator;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.KeyValueTextInputFormat;
import org.apache.hadoop.mapred.MapReduceBase;
import org.apache.hadoop.mapred.Mapper;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.TextInputFormat;
import org.apache.hadoop.mapred.TextOutputFormat;
import org.apache.hadoop.mapred.lib.IdentityReducer;
import org.apache.hadoop.mapred.lib.MultipleOutputFormat;
import org.apache.hadoop.mapred.lib.MultipleTextOutputFormat;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.FileOutputFormat;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

import com.google.inject.Key;
import com.sun.tracing.dtrace.ArgsAttributes;

public class MultiFile extends Configured implements Tool {

    public static class MapClass extends MapReduceBase implements Mapper<LongWritable, Text, NullWritable, Text>{

        public void map(LongWritable key,Text value,OutputCollector<NullWritable, Text> output,Reporter reporter)throws IOException{

            //System.out.println(value.toString());
            output.collect(NullWritable.get(), value);
        }
    }

    public static class PartitionByCountryMTOF extends MultipleTextOutputFormat<NullWritable, Text>{

        private static int K=0;
        @Override
        protected String generateFileNameForKeyValue(NullWritable key, Text value, String name) {
            // TODO Auto-generated method stub
            if(K<10)System.out.println(name);
            K++;
            String fields []=value.toString().split(",",-1);
            String country=fields[4].substring(1, 3);
            return country+"/"+name;
        }


    }
    @Override
    public int run(String[] arg0) throws Exception {
        // TODO Auto-generated method stub

        Configuration configuration=getConf();

        JobConf job=new JobConf(configuration,MultiFile.class);

        FileInputFormat.setInputPaths(job, new Path(arg0[0]));
        FileOutputFormat.setOutputPath(job, new Path(arg0[1]));
        job.setJobName("MultiFile");
        job.setMapperClass(MapClass.class);

        job.setInputFormat(TextInputFormat.class);
        job.setOutputFormat(PartitionByCountryMTOF.class);
        job.setOutputKeyClass(NullWritable.class);
        job.setOutputValueClass(Text.class);
        job.setReducerClass(IdentityReducer.class);

        job.setNumReduceTasks(0);

        JobClient.runJob(job);

        return 0;
    }

    public static void main(String[] args) throws Exception{
        // TODO Auto-generated method stub
        //ToolRunner.run(conf, tool, args)
        int res=ToolRunner.run(new Configuration(), new MultiFile(), args);
        System.exit(res);

    }

}

这是横向拆分数据，那我想纵向拆分怎么办？比如我想把专利中时间有关的项放到一个文件，地理信息相关的放入另一个文件怎么办？Hadoop还提供了一个MultipleOutputs，它所采用的方法并不是给每条记录请求一个文件名，而是创建多个OutputCollector

public class MultiOutput extends Configured implements Tool {


    public static class MapClass extends MapReduceBase implements Mapper<LongWritable, Text, NullWritable, Text>{
        private MultipleOutputs multipleOutputs;
        private OutputCollector<NullWritable, Text> collector;

        public void configure(JobConf job){
            multipleOutputs=new MultipleOutputs(job);
        }
        public void map(LongWritable key,Text value,OutputCollector<NullWritable, Text> output,Reporter reporter)throws IOException{

            //System.out.println(value.toString());
            String arr []=value.toString().split(",",-1);
            String chrono=arr[0]+","+arr[1]+","+arr[2];
            String geo=arr[0]+","+arr[4]+","+arr[5];
            collector=multipleOutputs.getCollector("chrono", reporter);
            collector.collect(NullWritable.get(), new Text(chrono));
            collector=multipleOutputs.getCollector("geo", reporter);
            collector.collect(NullWritable.get(), new Text(geo));
        }
        @Override
        public void close() throws IOException {
            // TODO Auto-generated method stub
            multipleOutputs.close();
        }

    }


    @Override
    public int run(String[] arg0) throws Exception {
        // TODO Auto-generated method stub

        Configuration configuration=getConf();

        JobConf job=new JobConf(configuration,MultiOutput.class);

        FileInputFormat.setInputPaths(job, new Path(arg0[0]));
        FileOutputFormat.setOutputPath(job, new Path(arg0[1]));
        job.setJobName("MultiFile");
        job.setMapperClass(MapClass.class);

        job.setInputFormat(TextInputFormat.class);
        //job.setOutputFormat(PartitionByCountryMTOF.class);
        job.setOutputKeyClass(NullWritable.class);
        job.setOutputValueClass(Text.class);
        job.setReducerClass(IdentityReducer.class);

        job.setNumReduceTasks(0);

        MultipleOutputs.addNamedOutput(job,"chrono", TextOutputFormat.class, NullWritable.class,Text.class);
        MultipleOutputs.addNamedOutput(job, "geo", TextOutputFormat.class, NullWritable.class, Text.class);

        JobClient.runJob(job);

        return 0;
    }

    public static void main(String[] args) throws Exception{
        // TODO Auto-generated method stub
        //ToolRunner.run(conf, tool, args)
        int res=ToolRunner.run(new Configuration(), new MultiOutput(), args);
        System.exit(res);

    }

}