【学习历程】11 MapReduce之InputFormat详解

一、前言

      InputFormat是mapreduce当中用于处理数据输入的一个组件，是最顶级的一个抽象父类，主要用于解决各个地方的数据源的数据输入问题。FileInputFormat类是InputFormat的一个子类，如果需要操作hdfs上面的文件，基本上都是通过FileInputFormat类来实现的，我们可以通过FileInputFormat来实现各种格式的文件操作。

二、自定义InputFormat

      mapreduce框架当中已经提供了很多的文件输入类，用于处理文件数据的输入，如果提供的文件数据类不够用的话，我们可以通过自定义InputFormat来实现文件数据的输入。

2.1 问题描述：

      通过自定义InputFormat实现将小文件全部读取，然后输出成为一个SequenceFile格式的大文件，进行文件的合并。

2.2 程序

1. 自定义InputFormat

import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.*;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

import java.io.IOException;

public class MyInputFormat extends FileInputFormat<NullWritable, BytesWritable> {

    @Override
    public RecordReader<NullWritable, BytesWritable> createRecordReader(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
        MyRecordReader myRecordReader = new MyRecordReader();
        myRecordReader.initialize(inputSplit,taskAttemptContext);

        return myRecordReader;
    }

    //是否可切片，false为不可切片
    @Override
    protected boolean isSplitable(JobContext context, Path filename) {
        return false;
    }
}

2. 自定义RecordReader读取数据

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;

import java.io.IOException;

public class MyRecordReader extends RecordReader<NullWritable, BytesWritable> {
    //分片
    private FileSplit fileSplit;
    //Configuration
    private Configuration configuration;
    //当前recordReader读取的value
    private BytesWritable bytesWritable;
    //reader是否已经读取分片数据；默认为false,没有读
    private boolean flag = false;


    //初始化方法，主要对类的成员做初始化
    @Override
    public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
        this.fileSplit = (FileSplit)inputSplit;
        this.configuration = taskAttemptContext.getConfiguration();
        this.bytesWritable = new BytesWritable();
    }

    //判断是否有下一个k,v对，第一次来读切片的时候，有下一个key，因此可以读取切片数据
    @Override
    public boolean nextKeyValue() throws IOException, InterruptedException {
        if (!flag){
            //开始读取分片的数据
            long length = fileSplit.getLength();  //分片长度
            byte[] bytes = new byte[(int) length];  //建立一个分片长度的数组，存储分片内容

            Path path = fileSplit.getPath();
            FileSystem fileSystem = path.getFileSystem(configuration);

            FSDataInputStream open = fileSystem.open(path); //调用open方法获取输入流

            IOUtils.readFully(open,bytes,0,(int) length); //使用工具类将输入流数据copy到字节数组中去

            bytesWritable.set(bytes,0, (int) length); //把字节数组数据序列化为BytesWritable

            flag = true;
            return true; //表示第一次读可以读取k，v
        }
        return false; //表示第二次读时，没有可以读取的k，v对，因为就一个分片，一次就可以读完
    }

    //当前key
    @Override
    public NullWritable getCurrentKey() throws IOException, InterruptedException {
        return NullWritable.get();
    }
    //当前value
    @Override
    public BytesWritable getCurrentValue() throws IOException, InterruptedException {
        return bytesWritable;
    }
    //当前读取进度，没读就是0%，读了就是100%
    @Override
    public float getProgress() throws IOException, InterruptedException {
        return flag ? 1.0f : 0.0f;
    }

    //各类资源的关闭
    @Override
    public void close() throws IOException {

    }
}

3. 自定义mapper类

import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;

import java.io.IOException;

public class MyInputFormatMapper extends Mapper<NullWritable, BytesWritable, Text,BytesWritable> {
    @Override
    protected void map(NullWritable key, BytesWritable value, Context context) throws IOException, InterruptedException {
        FileSplit fileSplit = (FileSplit)context.getInputSplit();
        String name = fileSplit.getPath().getName(); //获取文件的名称

        context.write(new Text(name),value);
    }
}

4. 定义main方法

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

public class MyInputFormatMain extends Configured implements Tool {
    public static void main(String[] args) throws Exception {
        Configuration configuration = new Configuration();
        int run = ToolRunner.run(configuration, new MyInputFormatMain(), args);
        System.exit(run);
    }

    @Override
    public int run(String[] args) throws Exception {
        Configuration conf = super.getConf();
        Job job = Job.getInstance(conf, "mergeSmallFile");
        job.setJarByClass(MyInputFormatMain.class);

        job.setInputFormatClass(MyInputFormat.class);
        MyInputFormat.addInputPath(job,new Path("file:///E:\\大数据\\数据"));

        job.setMapperClass(MyInputFormatMapper.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(BytesWritable.class);

        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(BytesWritable.class);

        job.setOutputFormatClass(SequenceFileOutputFormat.class);
        SequenceFileOutputFormat.setOutputPath(job,new Path("file:///E:\\大数据\\数据\\out"));

        boolean b = job.waitForCompletion(true);

        return b?0:1;
    }
}