Hadoop(十五)MapReduce 输入类InputFormat（切片）案例

MapReduce在处理小文件时效率很低，但面对大量的小文件又不可避免，这个时候就需要相应的解决方案。 默认的输入格式为TextInputFormat，对于小文件，它是按照它的父类FileInputFormat的切片机制来切片的，也就是不管一个文件多小，独占一片

案例1

对于之前的wordcount案例来说，输入目录下一共有3个文件，这将开启3个reduceTask去执行

需求

我们需要将三个文本文件合并为一个序列化文件

输入三个文本文件，输出一个文件

自定义Inputformat类

需要实现两个方法

isSplitable()：是否可以切片，我们修改返回值为false不可切割。

createRecordReader：返回我们自定义的RecordReader对象。

自定义inputfomat

package selfinputformat;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

import java.io.IOException;

public class MyInputFormat extends FileInputFormat<Text, BytesWritable> {

    //设置文件不可切片,设置一个文件最多作为1片
    @Override
    protected boolean isSplitable(JobContext context, Path filename) {
        return false;
    }

    //设置读取文件的格式为自定义格式
    @Override
    public RecordReader<Text,BytesWritable> createRecordReader(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
        return new MyRecordReader();
    }
}

自定义RecordReader类

要修改两个地方：key和value。 Mapper类中的map方法需要有四个参数，其中的KEY_IN和VALUE_IN都是由RecordReader类来设置的，这里需要设置一下。

默认的RecordReader类的key为LongWritable类型，也就是一行数据对应的字节偏移量，这里设置key为文件名，也就是Text类型。

默认的RecordReader类的value为Text类型，也就是一行文本，设置value为文件名key对应的文件的二进制序列，也就是BytesWritable类型。 

package selfinputformat;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import java.io.IOException;

public class MyRecordReader extends RecordReader<Text,BytesWritable> {
    private Text key;
    private BytesWritable value;
    private String filename;
    private int length;
    private FileSystem fs;
    private Path path;
    private FSDataInputStream is;
    private boolean flag=true;
    @Override
    public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
        FileSplit fileSplit = (FileSplit) inputSplit;
        filename = fileSplit.getPath().getName();
        length = (int) fileSplit.getLength();
        path = fileSplit.getPath();
        //获取当前Job的配置对象
        Configuration conf = taskAttemptContext.getConfiguration();
        //获取当前Job使用的文件系统
        fs = FileSystem.get(conf);
        is = fs.open(path);
    }


    //文件的名称做为 key,文件的内容封装为BytesWritable类型的 value, 返回true
    @Override
    public boolean nextKeyValue() throws IOException, InterruptedException {
        //第一次调用nextKeyValue方法
        if (flag){
            //实例化对象
            if (key==null){
                key = new Text();
            }
            if (value==null){
                value = new BytesWritable();
            }

            //赋值将文件名封装到key中
            key.set(filename);
            //将文件的内容读取封装到value中
            byte[] content = new byte[ length];
            IOUtils.readFully(is,content,0,length);
            value.set(content,0,length);
            flag = false;
            return true;
        }
        //第二次调用直接执行 return false
        return false;
    }

    //返回当前读取到的key
    @Override
    public Text getCurrentKey() throws IOException, InterruptedException {
        return key;
    }

    //返回当前读取到的value
    @Override
    public BytesWritable getCurrentValue() throws IOException, InterruptedException {
        return value;
    }

    //返回读取切片的进度
    @Override
    public float getProgress() throws IOException, InterruptedException {
        return 0;
    }

    //关闭资源
    @Override
    public void close() throws IOException {
        if (is != null){
            IOUtils.closeStream(is);
        }
        if (fs != null){
            fs.close();
        }
    }
}

Mapper类

在自定义RecordReader类中，已经设置输入的key为文件名，value设置为文件的二进制序列，所以这里直接将key和value写出即可，key的类型为Text，value的类型为BytesWritable

package selfinputformat;

import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import java.io.IOException;
public class SelfFileMapper  extends Mapper<Text, BytesWritable,Text,BytesWritable> {
    @Override
    protected void map(Text key, BytesWritable value, Context context) throws IOException, InterruptedException {
        context.write(key,value);
    }
}

Reducer类

public class SequenceFileReducer extends Reducer<Text, BytesWritable,Text,Text> {
    private Text OUT_VALUE = new Text();
    @Override
    protected void reduce(Text key, Iterable<BytesWritable> values, Context context) throws IOException, InterruptedException {
 
        String value = values.toString();
        OUT_VALUE.set(value);
        context.write(key,OUT_VALUE);
 
    }
}

启动类

import com.lyh.mapreduce.MaxTemp.MaxTempMapper;
import com.lyh.mapreduce.MaxTemp.MaxTempReducer;
import com.lyh.mapreduce.MaxTemp.MaxTempRunner;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
 
public class MySequenceFileRunner extends Configured implements Tool {
    public static void main(String[] args) throws Exception {
        ToolRunner.run(new Configuration(),new MySequenceFileRunner(),args);
    }
 
    @Override
    public int run(String[] args) throws Exception {
        //1.获取job
        Configuration conf = new Configuration();
        Job job = Job.getInstance(conf, "my sequence file demo");
 
        //2.配置jar包路径
        job.setJarByClass(MySequenceFileRunner.class);
 
        //3.关联mapper和reducer
        job.setMapperClass(SequenceFileMapper.class);
        job.setReducerClass(SequenceFileReducer.class);
 
        //4.设置map、reduce输出的k、v类型
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(BytesWritable.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(BytesWritable.class);
 
        //设置切片机制为我们自定义的切片机制
        job.setInputFormatClass(MyInputFormat.class);
 
        //5.设置统计文件输入的路径,将命令行的第一个参数作为输入文件的路径
        FileInputFormat.setInputPaths(job,new Path("D:\\MapReduce_Data_Test\\myinputformat\\input"));
        //6.设置结果数据存放路径,将命令行的第二个参数作为数据的输出路径
        FileOutputFormat.setOutputPath(job,new Path("D:\\MapReduce_Data_Test\\myinputformat\\output1"));
        return job.waitForCompletion(true) ? 0 : 1;//verbose：是否监控并打印job的信息
    }
}

打成jar包，运行

hadoop jar /hadoopmapreduce-1.0-SNAPSHOT.jar selfinputformat.JobMain /wcinput/ /my_wcoutputone

结果合并成一个文件

案例2

数据文件 studentscore.txt,可以看到三行为一组数据，那么常用的inputfomat就不起作用了，需要自定义

wjm
shuxue 91
yinyu 88
lilie
shuxue 73
yinyu 27
fuhu
shuxue 37
yinyu 90

 代码如下：

Reader

package selfinputformat1;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import org.apache.hadoop.util.LineReader;

import java.io.IOException;
import java.net.URI;

public class MyRecordReader extends RecordReader<Text, Text> {

    private LineReader reader;
    private Text key;
    private Text value;
    private long length;
    private float pos = 0;
    private static final byte[] blank = new Text(" ").getBytes();

    // 初始化方法，在初始化的时候会被调用一次
    // 一般会利用这个方法来获取一个实际的流用于读取数据
    @Override
    public void initialize(InputSplit split, TaskAttemptContext context) throws IOException {
        // 转化
        FileSplit fileSplit = (FileSplit) split;
        // 获取切片所存储的位置
        Path path = fileSplit.getPath();
        // 获取切片大小
        length = fileSplit.getLength();
        // 连接HDFS
        FileSystem fs = FileSystem.get(URI.create(path.toString()), context.getConfiguration());
        // 获取实际用于读取数据的输入流
        FSDataInputStream in = fs.open(path);
        // 获取到的输入流是一个字节流，要处理的文件是一个字符文件
        // 考虑将字节流包装成一个字符流，最好还能够按行读取
        reader = new LineReader(in);
    }

    // 判断是否有下一个键值对要交给map方法来处理
    // 试着读取文件。如果读取到了数据，那么说明有数据要交给map方法处理，此时返回true
    // 反之，如果没有读取到数据，那么说明所有的数据都处理完了，此时返回false
    @Override
    public boolean nextKeyValue() throws IOException {
        // 构建对象来存储数据
        key = new Text();
        value = new Text();
        Text tmp = new Text();
        // 读取第一行数据
        // 将读取到的数据放到tmp中
        // 返回值表示读取到的字节个数
        if (reader.readLine(tmp) <= 0) return false;
        key.set(tmp.toString());
        pos += tmp.getLength();
        // 读取第二行数据
        if (reader.readLine(tmp) <= 0) return false;
        value.set(tmp.toString());
        pos += tmp.getLength();
        // 读取第三行数据
        if (reader.readLine(tmp) <= 0) return false;
        value.append(blank, 0, blank.length);
        value.append(tmp.getBytes(), 0, tmp.getLength());
        pos += tmp.getLength();
        // key = wjm
        // value = shuxue 90 yinyu 98
        return true;
    }

    // 获取键
    @Override
    public Text getCurrentKey() {
        return key;
    }

    // 获取值
    @Override
    public Text getCurrentValue() {
        return value;
    }

    // 获取执行进度
    @Override
    public float getProgress() {
        return pos / length;
    }

    @Override
    public void close() throws IOException {
        if (reader != null)
            reader.close();
    }
}

 MyInputFormat

package selfinputformat1;


import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import org.apache.hadoop.util.LineReader;

import java.io.IOException;
import java.net.URI;

// 泛型表示读取出来的数据类型
public class MyInputFormat extends FileInputFormat<Text, Text> {
    @Override
    public RecordReader<Text, Text> createRecordReader(InputSplit split, TaskAttemptContext context) {
        return new MyRecordReader();
    }
}

 MyselfReducer

package selfinputformat1;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

public class JobMain extends Configured implements Tool {
    public static void main(String[] args) throws Exception {
        ToolRunner.run(new Configuration(),new JobMain(),args);
    }

    @Override
    public int run(String[] args) throws Exception {
        //1.获取job
        Configuration conf = new Configuration();
        Job job = Job.getInstance(conf, "my sequence file demo");

        //2.配置jar包路径
        job.setJarByClass(JobMain.class);

        //3.关联mapper和reducer
        job.setMapperClass(SelfFileMapper.class);
        job.setReducerClass(MyselfReducer.class);

        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);

        // 指定输入格式类
        job.setInputFormatClass(MyInputFormat.class);


        FileInputFormat.addInputPath(job,new Path(args[0]));
        FileOutputFormat.setOutputPath(job,new Path(args[1]));

        return job.waitForCompletion(true) ? 0 : 1;//verbose：是否监控并打印job的信息
    }
}

package selfinputformat1;

import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;

public class MyselfReducer extends Reducer<Text, IntWritable, Text, IntWritable> {
    @Override
    protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
        int sum = 0;
        for (IntWritable value : values) {
            sum += value.get();
        }
        context.write(key, new IntWritable(sum));
    }
}

 SelfFileMapper

package selfinputformat1;

import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;

public class SelfFileMapper  extends Mapper<Text, Text, Text, IntWritable> {
    @Override
    protected void map(Text key, Text value, Context context) throws IOException, InterruptedException {
        // key = tom
        // value = math 90 english 98
        // 拆分数据
        String[] arr = value.toString().split(" ");
        context.write(key, new IntWritable(Integer.parseInt(arr[1])));
        context.write(key, new IntWritable(Integer.parseInt(arr[3])));
    }
}

 JobMain

package selfinputformat1;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

public class JobMain extends Configured implements Tool {
    public static void main(String[] args) throws Exception {
        ToolRunner.run(new Configuration(),new JobMain(),args);
    }

    @Override
    public int run(String[] args) throws Exception {
        //1.获取job
        Configuration conf = new Configuration();
        Job job = Job.getInstance(conf, "my sequence file demo");

        //2.配置jar包路径
        job.setJarByClass(JobMain.class);

        //3.关联mapper和reducer
        job.setMapperClass(SelfFileMapper.class);
        job.setReducerClass(MyselfReducer.class);

        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);

        // 指定输入格式类
        job.setInputFormatClass(MyInputFormat.class);


        FileInputFormat.addInputPath(job,new Path(args[0]));
        FileOutputFormat.setOutputPath(job,new Path(args[1]));

        return job.waitForCompletion(true) ? 0 : 1;//verbose：是否监控并打印job的信息
    }
}

 运行代码

hadoop jar /hadoopmapreduce-1.0-SNAPSHOT.jar selfinputformat1.JobMain /studentscore.txt /my_wcoutputone1

 输出结果