自定义OutputFormat--Hadoop

1.首先介绍OutputFormat类的作用：
1）验证输出规范的有效性
2）为在输出文件进行写操作的RecordWriter实现类提供一个实例。
RecordWriter用于将输出的值与键类型的Java实例转化成原始字节。

2.主要讨论如何将文本文件转化成XML文件
输出示例

<recs>
 <rec>
   <key>0</key>
   <year>1988</year>
 </rec>
 <rec>
   <key>98</key>
   <year>1998<year>
 </rec>
</recs>

3.主程序代码

import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

public class OutputFormatTest extends Configured implements Tool{

    public static class TextToXMLConversionMapper extends Mapper<LongWritable,Text,LongWritable,Text>{
        public void map(LongWritable key,Text value,Context context) throws IOException, InterruptedException{
                if(value.toString().contains(" "))
                context.write(key, value);
        }
    }

    @Override
    public int run(String[] args) throws Exception {
        // TODO Auto-generated method stub
        Job job =Job.getInstance(getConf());
        job.setJarByClass(OutputFormatTest.class);
        job.setInputFormatClass(TextInputFormat.class);
        job.setOutputFormatClass(XMLOutputFormat.class);
//      job.setMapOutputKeyClass(LongWritable.class);
//      job.setMapOutputValueClass(Text.class);
        job.setOutputKeyClass(LongWritable.class);
        job.setOutputValueClass(Text.class);
        job.setMapperClass(TextToXMLConversionMapper.class);
        FileInputFormat.setInputPaths(job, new Path(args[0]));
        FileOutputFormat.setOutputPath(job, new Path(args[1]));
        job.waitForCompletion(true);
        return job.isSuccessful()?0:1;
    }
    public static void main(String args[]) throws Exception{
        int rex = ToolRunner.run(new Configuration(),new OutputFormatTest() , args);
        System.exit(rex);
    }
}

4.自定义OutputFormat类代码即XMLOutputFormat类

package com.input.output;

import java.io.DataOutputStream;
import java.io.IOException;

import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class XMLOutputFormat extends FileOutputFormat<LongWritable,Text>{

    protected static class XMLRecordWriter extends RecordWriter<LongWritable,Text>{

        private DataOutputStream out;
        public XMLRecordWriter(DataOutputStream out) throws IOException{
            this.out=out;
            out.writeBytes("<recs>\n");
        }
        private void writeTag(String tag,String value) throws IOException{
            out.writeBytes("<"+tag+">"+value+"</"+tag+">\n");
        }
        @Override
        public void close(TaskAttemptContext arg0) throws IOException, InterruptedException {
            // TODO Auto-generated method stub
            try{
                out.writeBytes("</recs>\n");
            }finally{
                out.close();
            }

        }

        @Override
        public void write(LongWritable key, Text value) throws IOException, InterruptedException {
            // TODO Auto-generated method stub
            out.writeBytes("<rec>");
            this.writeTag("key", Long.toString(key.get()));
            String contents[] =value.toString().split(",");
            this.writeTag("year", contents[0]);
            out.writeBytes("</rec>");
        }   
    }

    @Override
    public RecordWriter<LongWritable, Text> getRecordWriter(TaskAttemptContext job)
            throws IOException, InterruptedException {
        // TODO Auto-generated method stub
        String extension = "data.xml";
        Path file = getDefaultWorkFile(job,extension);
        FileSystem fs = file.getFileSystem(job.getConfiguration());
        FSDataOutputStream fileout = fs.create(file,false);
        return new XMLRecordWriter(fileout);
    }

}

5.输出结果的处理过程如下：
1）Hadoop的框架通过调用XMLOutputFormat类（OutputFormat的一个实现）实例的getRecordWriter()的方法，获得XMLRecordWriter实例的引用。
2）RecordWriter 配置输出的参数，包括文件名的前缀和扩展名。扩展名会在代码列表里提供。文件的默认前缀是part，该值