题目:日志文件分类读写
现有一个系统日志文件,需要对WARN、INFO、ERROR级别的日志进行分析,将含有WARN、INFO、ERROR的日志分别归类到不同的文件。日志内容详见catalina.out.
=============================================================
public class LogMapper extends Mapper<LongWritable, Text,Text, Text> {
public static final String LEVEL_INFO = "INFO";
public static final String LEVEL_ERROR = "ERROR";
public static final String LEVEL_WARN = "WARN";
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
if(value==null||value.toString().split("\\s").length < 3){
return;
}
String LogLevel = value.toString().split("\\s")[2];
if(LogLevel.equals(LEVEL_INFO)){
context.write(new Text(LEVEL_INFO),value);
}else if(LogLevel.equals(LEVEL_ERROR)){
context.write(new Text(LEVEL_ERROR),value);
}else if(LogLevel.equals(LEVEL_WARN)){
context.write(new Text(LEVEL_WARN),value);
}
}
}
=============================================================
public class LogReducer extends Reducer<Text,Text,Text, Text> {
@Override
protected void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
for (Text next : values) {
context.write(key, next);
}
}
}
=============================================================
public class LogOutputFormat extends FileOutputFormat<Text, Text> {
public static final String INFO = "INFO";
public static final String ERROR = "ERROR";
public static final String WARN = "WARN";
@Override
public RecordWriter<Text, Text> getRecordWriter(TaskAttemptContext job) throws IOException, InterruptedException {
Configuration configuration = job.getConfiguration();
FileSystem fs = FileSystem.get(configuration);
String outPath = configuration.get("mapred.output.dir");
Path INFO = new Path(outPath+"/INFO");
Path ERROR = new Path(outPath+"/ERROR");
Path WARN = new Path(outPath+"/WARN");
FSDataOutputStream INFOOut = fs.create(INFO);
FSDataOutputStream ERROROut = fs.create(ERROR);
FSDataOutputStream WARNOut = fs.create(WARN);
return new MyRecordWriter(INFOOut, ERROROut,WARNOut);
}
static class MyRecordWriter extends RecordWriter<Text, Text>{
FSDataOutputStream INFOOut = null;
FSDataOutputStream ERROROut = null;
FSDataOutputStream WARNOut = null;
public MyRecordWriter(FSDataOutputStream INFOOut,
FSDataOutputStream ERROROut,
FSDataOutputStream WARNOut) {
super();
this.INFOOut = INFOOut;
this.ERROROut = ERROROut;
this.WARNOut = WARNOut;
}
@Override
public void write(Text key, Text value) throws IOException,
InterruptedException {
String level = key.toString();
if(level.equals(INFO)){
INFOOut.write(value.getBytes());
}else if(level.equals(ERROR)){
ERROROut.write(value.getBytes());
}else if(level.equals(WARN)){
WARNOut.write(value.getBytes());
}
}
@Override
public void close(TaskAttemptContext context) throws IOException,
InterruptedException {
IOUtils.closeStream(INFOOut);
IOUtils.closeStream(ERROROut);
IOUtils.closeStream(WARNOut);
}
}
}
=============================================================
public class MapReduceTest1Main {
public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
Configuration conf = new Configuration();
conf.set("fs.defaultFS","file:///");
Job job = Job.getInstance(conf);
job.setJarByClass(LogMain.class);
job.setMapperClass(LogMapper.class);
job.setReducerClass(LogReducer.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(Text.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
job.setOutputFormatClass(LogOutputFormat.class);
FileInputFormat.addInputPath(job,new Path("E:\\log\\input"));
FileOutputFormat.setOutputPath(job,new Path("E:\\log\\output"));
job.waitForCompletion(true);
}
}