MapReduce增强06(自定义inputFormat合并小文件)

1.1 需求

无论hdfs还是mapreduce，对于小文件都有损效率，实践中，又难免面临处理大量小文件的场景，此时，就需要有相应解决方案

1.2 分析

小文件的优化有以下3种方式：

• 在数据采集的时候，就将小文件或小批数据合成大文件再上传HDFS
• 在业务处理之前，在HDFS上使用mapreduce程序对小文件进行合并
• 在mapreduce处理时，可采用combineInputFormat提高效率

1.3 实现

本次实现的是上述第二种方式
程序的核心机制：

• 自定义一个InputFormat
• 改写RecordReader，实现一次读取一个完整文件封装为KV
• 在输出时使用SequenceFileOutPutFormat输出合并文件

1.4 自定义InputFromat

public class WholeFileInputFormat extends FileInputFormat<NullWritable, BytesWritable>{
   /*
   直接返回文件不可切割,保证一个文件是一个完整的一行
    */
   @Override
   protected boolean isSplitable(JobContext context, Path file) {
      return false;
   }
   @Override
   public RecordReader<NullWritable, BytesWritable> createRecordReader(InputSplit split, TaskAttemptContext context) throws IOException,InterruptedException {
      WholeFileRecordReader reader = new WholeFileRecordReader();
      reader.initialize(split, context);
      return reader;
   }
}

1.5 自定义RecordReader

/**
 * 
 * RecordReader的核心工作逻辑：
 * 通过nextKeyValue()方法去读取数据构造将返回的key   value
 * 通过getCurrentKey 和 getCurrentValue来返回上面构造好的key和value
 * 
 * 
 * @author
 *
 */
public class WholeFileRecordReader extends RecordReader<NullWritable, BytesWritable> {
   private FileSplit fileSplit;
   private Configuration conf;
   private BytesWritable value = new BytesWritable();
   private boolean processed = false;
   @Override
   public void initialize(InputSplit split, TaskAttemptContext context)
         throws IOException, InterruptedException {
      this.fileSplit = (FileSplit) split;
      this.conf = context.getConfiguration();
   }
   @Override
   public boolean nextKeyValue() throws IOException, InterruptedException {
      if (!processed) {
         byte[] contents = new byte[(int) fileSplit.getLength()];
         Path file = fileSplit.getPath();
         FileSystem fs = file.getFileSystem(conf);
         FSDataInputStream in = null;
         try {
            in = fs.open(file);
            IOUtils.readFully(in, contents, 0, contents.length);
            value.set(contents, 0, contents.length);
         } finally {
            IOUtils.closeStream(in);
         }
         processed = true;
         return true;
      }
      return false;
   }
   @Override
   public NullWritable getCurrentKey() throws IOException,
         InterruptedException {
      return NullWritable.get();
   }
   @Override
   public BytesWritable getCurrentValue() throws IOException,
         InterruptedException {
      return value;
   }
   @Override
   public float getProgress() throws IOException {
      return processed ? 1.0f : 0.0f;
   }
   @Override
   public void close() throws IOException {
   }
}

1.6 定义mapreduce处理流程

public class SmallFilesToSequenceFileConverter extends Configured implements Tool {
   static class SequenceFileMapper extends
         Mapper<NullWritable, BytesWritable, Text, BytesWritable> {
      private Text filenameKey;
      @Override
      protected void setup(Context context) throws IOException,
            InterruptedException {
         InputSplit split = context.getInputSplit();
         Path path = ((FileSplit) split).getPath();
         filenameKey = new Text(path.toString());
      }
      @Override
      protected void map(NullWritable key, BytesWritable value,
            Context context) throws IOException, InterruptedException {
         context.write(filenameKey, value);
      }
   }
   @Override
   public int run(String[] args) throws Exception {
      Configuration conf = new Configuration();
      Job job = Job.getInstance(conf,"combine small files to sequencefile");
      job.setJarByClass(SmallFilesToSequenceFileConverter.class);
      job.setInputFormatClass(WholeFileInputFormat.class);
      WholeFileInputFormat.addInputPath(job,new Path("file:///F:\\自定义inputformat_小文件合并\\input"));
      job.setOutputFormatClass(SequenceFileOutputFormat.class);
      SequenceFileOutputFormat.setOutputPath(job,new Path("file:///F:\\自定义inputformat_小文件合并\\output"));
      job.setOutputKeyClass(Text.class);
      job.setOutputValueClass(BytesWritable.class);
      job.setMapperClass(SequenceFileMapper.class);
      return job.waitForCompletion(true) ? 0 : 1;
   }
   public static void main(String[] args) throws Exception {
      int exitCode = ToolRunner.run(new SmallFilesToSequenceFileConverter(),
            args);
      System.exit(exitCode);
      
   }
}