MapReduce练习-根据关键词进行分类

题目：日志文件分类读写
现有一个系统日志文件，需要对WARN、INFO、ERROR级别的日志进行分析，将含有WARN、INFO、ERROR的日志分别归类到不同的文件。日志内容详见catalina.out.

=============================================================

public class LogMapper extends Mapper<LongWritable, Text,Text, Text> {

    public static final String LEVEL_INFO = "INFO";
    public static final String LEVEL_ERROR = "ERROR";
    public static final String LEVEL_WARN = "WARN";

    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {

        if(value==null||value.toString().split("\\s").length < 3){
            return;
        }

        String LogLevel = value.toString().split("\\s")[2];
        if(LogLevel.equals(LEVEL_INFO)){
            context.write(new Text(LEVEL_INFO),value);
        }else if(LogLevel.equals(LEVEL_ERROR)){
            context.write(new Text(LEVEL_ERROR),value);
        }else if(LogLevel.equals(LEVEL_WARN)){
            context.write(new Text(LEVEL_WARN),value);
        }

    }
}

=============================================================

public class LogReducer extends Reducer<Text,Text,Text, Text> {

    @Override
    protected void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
        for (Text next : values) {
            context.write(key, next);
        }
    }
}

=============================================================

public class LogOutputFormat extends FileOutputFormat<Text, Text> {

    public static final String INFO = "INFO";
    public static final String ERROR = "ERROR";
    public static final String WARN = "WARN";

    @Override
    public RecordWriter<Text, Text> getRecordWriter(TaskAttemptContext job) throws IOException, InterruptedException {
        Configuration configuration = job.getConfiguration();
        FileSystem fs = FileSystem.get(configuration);
        String outPath = configuration.get("mapred.output.dir");
        Path INFO = new Path(outPath+"/INFO");
        Path ERROR = new Path(outPath+"/ERROR");
        Path WARN = new Path(outPath+"/WARN");
        FSDataOutputStream INFOOut = fs.create(INFO);
        FSDataOutputStream ERROROut = fs.create(ERROR);
        FSDataOutputStream WARNOut = fs.create(WARN);
        return new MyRecordWriter(INFOOut, ERROROut,WARNOut);
    }
    static class MyRecordWriter extends RecordWriter<Text, Text>{
        FSDataOutputStream INFOOut = null;
        FSDataOutputStream ERROROut = null;
        FSDataOutputStream WARNOut = null;
        public MyRecordWriter(FSDataOutputStream INFOOut,
                              FSDataOutputStream ERROROut,
                              FSDataOutputStream WARNOut) {
            super();
            this.INFOOut = INFOOut;
            this.ERROROut = ERROROut;
            this.WARNOut = WARNOut;
        }
        @Override
        public void write(Text key, Text value) throws IOException,
                InterruptedException {
            String level = key.toString();
            if(level.equals(INFO)){
                INFOOut.write(value.getBytes());
            }else if(level.equals(ERROR)){
                ERROROut.write(value.getBytes());
            }else if(level.equals(WARN)){
                WARNOut.write(value.getBytes());
            }
        }
        @Override
        public void close(TaskAttemptContext context) throws IOException,
                InterruptedException {
            IOUtils.closeStream(INFOOut);
            IOUtils.closeStream(ERROROut);
            IOUtils.closeStream(WARNOut);
        }
    }
}

=============================================================

public class MapReduceTest1Main {

    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {

        Configuration conf = new Configuration();
        conf.set("fs.defaultFS","file:///");
        Job job = Job.getInstance(conf);
        job.setJarByClass(LogMain.class);
        job.setMapperClass(LogMapper.class);
        job.setReducerClass(LogReducer.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(Text.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(Text.class);
        job.setOutputFormatClass(LogOutputFormat.class);
        FileInputFormat.addInputPath(job,new Path("E:\\log\\input"));
        FileOutputFormat.setOutputPath(job,new Path("E:\\log\\output"));

        job.waitForCompletion(true);

    }

}