18.自定义Inputformat与SequenceFile

需求：

将一个文件夹里的几个小文件读入并合并，输出为 ： 文件路径+文件内容

代码:

public class Fcinputformat extends FileInputFormat<NullWritable, BytesWritable> {
    @Override
    protected boolean isSplitable(JobContext context, Path filename) {
        //不切原来文件
        return false;
    }

    @Override
    public RecordReader<NullWritable, BytesWritable> createRecordReader(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {

      FcRecordReader fc =  new FcRecordReader();
        return fc;
    }
}

-----------------------------------------------------------
public class FcRecordReader extends RecordReader<NullWritable, BytesWritable> {
    boolean isProcess = false;
    FileSplit sp;
    Configuration conf;
    BytesWritable value = new BytesWritable();

    public void initialize(InputSplit inputSplit, TaskAttemptContext Context)  {
        this.sp = (FileSplit) inputSplit;
        conf = Context.getConfiguration();
    }

    public boolean nextKeyValue() throws IOException {

        if (!isProcess){
                 FSDataInputStream fis ;
                 FileSystem fs ;

                //1.根据切片长度获得缓冲区
                byte [] bur = new byte[(int)sp.getLength()];
                //2.获得路径
                Path path = sp.getPath();
                //3.通过路径获得文件系统
                fs = path.getFileSystem(conf);
                //4.通过文件系统获得输入流
                fis = fs.open(path);
                //5.拷贝流
                IOUtils.readFully(fis,bur,0,bur.length);
                //6.关闭流

                value.set(bur, 0, bur.length);


                IOUtils.closeStream(fis);
                IOUtils.closeStream(fs);



            isProcess = true;

            return true;
        }

        return false;
    }

    public NullWritable getCurrentKey()  {
        return NullWritable.get();
    }

    public BytesWritable getCurrentValue()  {
        return value;
    }

    public float getProgress() {
        return 0;
    }

    public void close()  {

    }
}
----------------------------------------------------
public class SquenceDrive {

    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
        Configuration conf = new Configuration();
        Job job = Job.getInstance(conf);

        job.setJarByClass(SquenceDrive.class);



        job.setMapperClass(SquenceMapper.class);
        job.setReducerClass(SquenceReducer.class);

        job.setInputFormatClass(Fcinputformat.class);
        job.setOutputFormatClass(SequenceFileOutputFormat.class);
//        job.setOutputFormatClass(TextOutputFormat.class);


        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(BytesWritable.class);

        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(BytesWritable.class);

        FileInputFormat.setInputPaths(job,new Path("B:/测试数据/"));
        FileOutputFormat.setOutputPath(job,new Path("B:/测试数据/out"));

        boolean b = job.waitForCompletion(true);

        System.out.println(b);
    }
}
-----------------------------------------------------------------------------------------
public class SquenceMapper extends Mapper<NullWritable, BytesWritable, Text, BytesWritable> {
        Text k = new Text();
    @Override
    protected void setup(Context context) throws IOException, InterruptedException {
        FileSplit sp = (FileSplit) context.getInputSplit();
        Path path = sp.getPath();
        k.set(path.toString());
    }

    @Override
    protected void map(NullWritable key, BytesWritable value, Context context) throws IOException, InterruptedException {
        context.write(k,value);
    }
}
--------------------------------------------------------------------
public class SquenceReducer extends Reducer<Text, BytesWritable,Text, BytesWritable> {
    @Override
    protected void reduce(Text key, Iterable<BytesWritable> values, Context context) throws IOException, InterruptedException {
        for (BytesWritable v : values){
            context.write(key,v);
        }
    }
}

输入结果:

上面包名不能去掉，用的不多，主要熟悉下Inputformat的过程和写法！

SequenceFile数据结构

https://www.cnblogs.com/yangsy0915/p/5559678.html