Hadoop实现自定义InputFormat按单个文件Map

在hadoop中，通常的处理经常用到的是读取某个数据文件，按行或者按一些简单的结构进行文件读取。但实际使用中，许多情况下，我们的输入文件不一定是简单的txt文件，可能是流文件、视频文件等等，这里就需要自定义InputFormat来处理输入文件。


下面贴出一个使用了自定义的WholeFileInputFormat的例子来实现一次读取某个完整文件给map函数的例子，与之类似还有其他的方法

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.examples.SecondarySort.Reduce;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Mapper.Context;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class WholeFileTest {
	public static class mapper extends Mapper
	{
		protected void map(Text key, BytesWritable value, Context context)
		{
			try {
				context.write(key, new Text(value.getBytes()));
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			} catch (InterruptedException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}
	}
	public static class reducer extends Reducer
	{
		public void reduce(Text key, Iterable values, Context context)
		{
			for(Text t : values)
			{
				try {
					context.write(key, t);
				} catch (IOException e) {
					// TODO Auto-generated catch block
					e.printStackTrace();
				} catch (InterruptedException e) {
					// TODO Auto-generated catch block
					e.printStackTrace();
				}
			}
		}
	}
	public static void main(String[] args) throws Exception
	{
		Configuration conf = new Configuration();
		Job job = new Job(conf,"wholeFile");
		job.setJarByClass(WholeFileTest.class);
		job.setMapperClass(mapper.class);
		job.setReducerClass(reducer.class);
		job.setInputFormatClass(WholeFileInputformat.class);
		job.setOutputKeyClass(Text.class);
		job.setOutputValueClass(Text.class);
		FileInputFormat.addInputPath(job, new Path("input"));
		FileOutputFormat.setOutputPath(job, new Path("output"));
		System.out.println(job.waitForCompletion(true)?0:1);
	}
}

class WholeFileInputformat extends FileInputFormat
{

	@Override
	public RecordReader createRecordReader(
			InputSplit arg0, TaskAttemptContext arg1) throws IOException,
			InterruptedException {
		// TODO Auto-generated method stub
		return new WholeFileRecordReader();
	}
	@Override  
	protected boolean isSplitable(JobContext context, Path filename) {  
	    // TODO Auto-generated method stub  
	    return false;  
	} 
}

class WholeFileRecordReader extends RecordReader
{
	private FileSplit fileSplit;
	private FSDataInputStream fis;

	private Text key=null;
	private BytesWritable value = null;

	private boolean processed = false;

	@Override
	public void close() throws IOException {
		// TODO Auto-generated method stub

	}

	@Override
	public Text getCurrentKey() throws IOException, InterruptedException {
		// TODO Auto-generated method stub
		return this.key;
	}

	@Override
	public BytesWritable getCurrentValue() throws IOException,
			InterruptedException {
		// TODO Auto-generated method stub
		return this.value;
	}

	@Override
	public float getProgress() throws IOException, InterruptedException {
		// TODO Auto-generated method stub
		return processed? fileSplit.getLength():0;
	}

	@Override
	public void initialize(InputSplit inputSplit, TaskAttemptContext tacontext)
			throws IOException, InterruptedException {
		// TODO Auto-generated method stub
		fileSplit = (FileSplit)inputSplit;
		Configuration job = tacontext.getConfiguration();
		Path file = fileSplit.getPath();
		FileSystem fs = file.getFileSystem(job);
		fis = fs.open(file);
	}

	@Override
	public boolean nextKeyValue() throws IOException, InterruptedException {
		// TODO Auto-generated method stub
		if(key==null)
		{
			key = new Text();
		}
		if(value==null)
		{
			value = new BytesWritable();
		}
		if(!processed)
		{
			byte[] content = new byte[(int)fileSplit.getLength()];
			Path file = fileSplit.getPath();
			System.out.println(file.getName());
			key.set(file.getName());
			try{
				IOUtils.readFully(fis, content, 0, content.length);
				value.set(new BytesWritable(content));
			}catch(IOException e)
			{
				e.printStackTrace();
			}finally{
				IOUtils.closeStream(fis);
			}
			processed = true;
			return true;               //return true表示这次inputformat还没有结束，会有下一对keyvalue产生
		}
		return false;                      //return false表示这次inputformat结束了
	}
}

 



更多的有关自定义inputformat和outputformat的信息可以参考下面的地址


http://www.cnblogs.com/zhangchaoyang/articles/2649660.html