在hadoop中,通常的处理经常用到的是读取某个数据文件,按行或者按一些简单的结构进行文件读取。但实际使用中,许多情况下,我们的输入文件不一定是简单的txt文件,可能是流文件、视频文件等等,这里就需要自定义InputFormat来处理输入文件。
下面贴出一个使用了自定义的WholeFileInputFormat的例子来实现一次读取某个完整文件给map函数的例子,与之类似还有其他的方法
更多的有关自定义inputformat和outputformat的信息可以参考下面的地址
http://www.cnblogs.com/zhangchaoyang/articles/2649660.html
下面贴出一个使用了自定义的WholeFileInputFormat的例子来实现一次读取某个完整文件给map函数的例子,与之类似还有其他的方法
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.examples.SecondarySort.Reduce;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Mapper.Context;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
public class WholeFileTest {
public static class mapper extends Mapper
{
protected void map(Text key, BytesWritable value, Context context)
{
try {
context.write(key, new Text(value.getBytes()));
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
public static class reducer extends Reducer
{
public void reduce(Text key, Iterable values, Context context)
{
for(Text t : values)
{
try {
context.write(key, t);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
}
public static void main(String[] args) throws Exception
{
Configuration conf = new Configuration();
Job job = new Job(conf,"wholeFile");
job.setJarByClass(WholeFileTest.class);
job.setMapperClass(mapper.class);
job.setReducerClass(reducer.class);
job.setInputFormatClass(WholeFileInputformat.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
FileInputFormat.addInputPath(job, new Path("input"));
FileOutputFormat.setOutputPath(job, new Path("output"));
System.out.println(job.waitForCompletion(true)?0:1);
}
}
class WholeFileInputformat extends FileInputFormat
{
@Override
public RecordReader createRecordReader(
InputSplit arg0, TaskAttemptContext arg1) throws IOException,
InterruptedException {
// TODO Auto-generated method stub
return new WholeFileRecordReader();
}
@Override
protected boolean isSplitable(JobContext context, Path filename) {
// TODO Auto-generated method stub
return false;
}
}
class WholeFileRecordReader extends RecordReader
{
private FileSplit fileSplit;
private FSDataInputStream fis;
private Text key=null;
private BytesWritable value = null;
private boolean processed = false;
@Override
public void close() throws IOException {
// TODO Auto-generated method stub
}
@Override
public Text getCurrentKey() throws IOException, InterruptedException {
// TODO Auto-generated method stub
return this.key;
}
@Override
public BytesWritable getCurrentValue() throws IOException,
InterruptedException {
// TODO Auto-generated method stub
return this.value;
}
@Override
public float getProgress() throws IOException, InterruptedException {
// TODO Auto-generated method stub
return processed? fileSplit.getLength():0;
}
@Override
public void initialize(InputSplit inputSplit, TaskAttemptContext tacontext)
throws IOException, InterruptedException {
// TODO Auto-generated method stub
fileSplit = (FileSplit)inputSplit;
Configuration job = tacontext.getConfiguration();
Path file = fileSplit.getPath();
FileSystem fs = file.getFileSystem(job);
fis = fs.open(file);
}
@Override
public boolean nextKeyValue() throws IOException, InterruptedException {
// TODO Auto-generated method stub
if(key==null)
{
key = new Text();
}
if(value==null)
{
value = new BytesWritable();
}
if(!processed)
{
byte[] content = new byte[(int)fileSplit.getLength()];
Path file = fileSplit.getPath();
System.out.println(file.getName());
key.set(file.getName());
try{
IOUtils.readFully(fis, content, 0, content.length);
value.set(new BytesWritable(content));
}catch(IOException e)
{
e.printStackTrace();
}finally{
IOUtils.closeStream(fis);
}
processed = true;
return true; //return true表示这次inputformat还没有结束,会有下一对keyvalue产生
}
return false; //return false表示这次inputformat结束了
}
}
更多的有关自定义inputformat和outputformat的信息可以参考下面的地址
http://www.cnblogs.com/zhangchaoyang/articles/2649660.html