过滤日志及自定义日志输出路径案例（自定义OutputFormat）

自定义OutputFormat

为了实现控制最终文件的输出路径，可以自定义OutputFormat。

要在一个mapreduce程序中根据数据的不同输出两类结果到不同目录，这类灵活的输出需求可以通过自定义outputformat来实现。

1）自定义OutputFormat步骤

（1）自定义一个类继承FileOutputFormat。

（2）改写recordwriter，具体改写输出数据的方法write()。

2）实操案例：

1）需求

过滤输入的log日志中是否包含atguigu

（1）包含atguigu的网站输出到e:/atguigu.log

（2）不包含atguigu的网站输出到e:/other.log

2）输入数据

log.txt

http://www.baidu.com
http://www.google.com
http://cn.bing.com
http://www.atguigu.com
http://www.sohu.com
http://www.sina.com
http://www.sin2a.com
http://www.sin2desa.com
http://www.sindsafa.com

输出预期：

atguigu.log

http://www.atguigu.com

other.log

http://cn.bing.com
http://www.baidu.com
http://www.google.com
http://www.sin2a.com
http://www.sin2desa.com
http://www.sina.com
http://www.sindsafa.com
http://www.sohu.com

代码实现

（1）自定义一个outputformat

package com.lzz.mapreduce.outputformat;

import java.io.IOException;

import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class FilterOutputFormat extends FileOutputFormat<Text, NullWritable>{

	@Override
	public RecordWriter<Text, NullWritable> getRecordWriter(TaskAttemptContext job)
			throws IOException, InterruptedException {
		// 创建一个RecordWriter
		return new FilterRecordWriter(job);
	}

}

（2）具体的写数据RecordWriter

package com.lzz.mapreduce.outputformat;

import java.io.IOException;

import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;

public class FilterRecordWriter extends RecordWriter<Text, NullWritable> {

	private FSDataOutputStream atguiguOut=null;
	private FSDataOutputStream otherOut=null;

	public FilterRecordWriter(TaskAttemptContext job)  {
		// 1 获取文件系统
				FileSystem fs;

				try {
					fs = FileSystem.get(job.getConfiguration());

					// 2 创建输出文件路径
					Path atguiguPath = new Path("g:/output/atguigu.log");
					Path otherPath = new Path("g:/output/other.log");

					// 3 创建输出流
					atguiguOut = fs.create(atguiguPath);
					otherOut = fs.create(otherPath);
				} catch (IOException e) {
					e.printStackTrace();
				}
	}

	@Override
	public void write(Text key, NullWritable value) throws IOException, InterruptedException {
		// 判断是否包含"atguigu"输出到不同文件
		if (key.toString().contains("atguigu")) {
			atguiguOut.write(key.toString().getBytes());
		}else {
			otherOut.write(key.toString().getBytes());
		}
	}

	@Override
	public void close(TaskAttemptContext context) throws IOException, InterruptedException {
		//关闭资源
		if(atguiguOut!=null) {
			atguiguOut.close();
		}if(otherOut!=null) {
			otherOut.close();
		}
	}

}

（3）编写FilterMapper

package com.lzz.mapreduce.outputformat;

import java.io.IOException;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

public class FilterMapper extends Mapper<LongWritable, Text, Text, NullWritable>{
	Text k=new Text();
	@Override
	protected void map(LongWritable key, Text value, Context context)
			throws IOException, InterruptedException {
		//1获取一行
//		http://www.baidu.com
//		http://www.google.com
		
		String line=value.toString();
		//2设置k
		k.set(line);
		//3输出
		context.write(k, NullWritable.get());
		
	}
}

（4）编写FilterReducer

package com.lzz.mapreduce.outputformat;

import java.io.IOException;

import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

public class FilterReducer extends Reducer<Text, NullWritable, Text, NullWritable>{
	@Override
	protected void reduce(Text key, Iterable<NullWritable> values,
			Context context) throws IOException, InterruptedException {
		context.write(key, NullWritable.get());
	}
}

（5）编写FilterDriver

package com.lzz.mapreduce.outputformat;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;



public class FilterDriver {
	public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
		args=new String[] {"g:/input/outputformat","g:/output"};
		
		Configuration configuration=new Configuration();
		Job job=Job.getInstance(configuration);
		
		job.setJarByClass(FilterDriver.class);
		
		job.setMapperClass(FilterMapper.class);
		job.setReducerClass(FilterReducer.class);
		
		job.setMapOutputKeyClass(Text.class);
		job.setMapOutputValueClass(NullWritable.class);
		
		job.setOutputKeyClass(Text.class);
		job.setOutputValueClass(NullWritable.class);
		
		//要将自定义的输出格式组件设置到job中
		job.setOutputFormatClass(FilterOutputFormat.class);
		
		FileInputFormat.setInputPaths(job, new Path(args[0]));
		//虽然我们自定义了outputformat,但是因为我们的outputformat继承自fileoutputformat
		//而fileoutformat要输出一个SUCCESS文件，所以，在这还得制定一个输出目录
		FileOutputFormat.setOutputPath(job, new Path(args[1]));
		
		boolean res=job.waitForCompletion(true);
		System.exit(res?0:1);
	}
}