MR程序案例 找出文件中提取需要的内容，并把文件输出成两个结果文件

public class LogDriver {
    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
        Configuration configuration = new Configuration();
        Job job = Job.getInstance(configuration);

        job.setJarByClass(LogDriver.class);
        job.setMapperClass(LogMapper.class);
        job.setReducerClass(LogReducer.class);

        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(NullWritable.class);

        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(NullWritable.class);

        //设置自定义输出类信息
        job.setOutputFormatClass(LogOutputFormat.class);

        FileInputFormat.setInputPaths(job,new Path("I:\\input\\inputoutputformat\\log.txt"));
        //虽然我们自定义了outputformat，但是因为我们的outputformat继承自fileoutputformat
        //而fileoutputformat要输出一个_SUCCESS文件，所以在这还得指定一个输出目录
        FileOutputFormat.setOutputPath(job,new Path("I:/output/LogOutput3/"));

        boolean b = job.waitForCompletion(true);
        System.exit(b?0:1);

    }
}

public class LogMapper extends Mapper<LongWritable, Text,Text, NullWritable> {
    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        //不做任何处理直接传给Reduce
        context.write(value,NullWritable.get());
    }
}

public class LogOutputFormat extends FileOutputFormat<Text, NullWritable> {

    @Override
    public RecordWriter<Text, NullWritable> getRecordWriter(TaskAttemptContext job) throws IOException, InterruptedException {
        LogRecordWriter logRecordWriter = new LogRecordWriter(job);
        return logRecordWriter;
    }
}

public class LogRecordWriter extends RecordWriter<Text, NullWritable> {

    private FSDataOutputStream ops1;
    private FSDataOutputStream ops2;

    public LogRecordWriter(TaskAttemptContext job) {
        //不要在构造器中抛异常！！
        try {
            FileSystem fs = FileSystem.get(job.getConfiguration());
            //使用文件系统对象创建两个输出流输出两个目录
            ops1 = fs.create(new Path("I:/outputLog1/atguigu.log"));
            ops2 = fs.create(new Path("I:/outputLog2/other.log"));
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    @Override
    public void write(Text key, NullWritable value) throws IOException, InterruptedException {
        String line = key.toString();
        if(line.contains("atguigu")){
            ops1.writeBytes(line + '\t');
        }else{
            ops2.writeBytes(line + '\t');
        }
    }


    @Override
    public void close(TaskAttemptContext context) throws IOException, InterruptedException {
//        ops1.close();
//        ops2.close();
        IOUtils.closeStreams(ops1,ops2);
    }
}

public class LogReducer  extends Reducer<Text, NullWritable,Text,NullWritable> {
    @Override
    protected void reduce(Text key, Iterable<NullWritable> values, Context context) throws IOException, InterruptedException {
        for (NullWritable value : values) {
            //也要输出相同的内容
            context.write(key,value);
        }
    }
}