无论hdfs还是mapreduce,对于小文件都有损效率,实践中,又难免面临处理大量小文件的场景,此时,就需要有相应解决方案。将多个小文件合并成一个文件SequenceFile,SequenceFile里面存储着多个文件,存储的形式为文件路径+名称为key,文件内容为value。
这其实就是重写inputformat改变它读取的规则,把几个文件合并到一个文件中去,来实现小文件合并.
package com.buba.mapreduce.wholefile;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import java.io.IOException;
public class WholeFileInputformat extends FileInputFormat<NullWritable, BytesWritable> {
//是否可以切割
@Override
protected boolean isSplitable(JobContext context, Path filename) {
return false;
}
@Override
public RecordReader<NullWritable, BytesWritable> createRecordReader(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
WholeRecordReader reader = new WholeRecordReader();
//调用初始化方法
reader.initialize(inputSplit,taskAttemptContext);
return reader;
}
}
package com.buba.mapreduce.wholefile;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import java.io.IOException;
public class WholeRecordReader extends RecordReader<NullWritable, BytesWritable> {
private BytesWritable value = new BytesWritable();
private FileSplit split;
private Configuration configuration;
private boolean isProcess = false;
//初始化方法
@Override
public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
//获取切片信息
this.split = (FileSplit) inputSplit;
//获取配置信息
configuration = taskAttemptContext.getConfiguration();
}
//具体业务逻辑
@Override
public boolean nextKeyValue() throws IOException, InterruptedException {
FSDataInputStream fis = null;
if(!isProcess){
try {
//按文件整体处理,读取
FileSystem fs = FileSystem.get(configuration);
//获取切片的路径
Path path = split.getPath();
//获取到切片的输入流
fis= fs.open(path);
byte[] buf = new byte[(int)split.getLength()];
//读取数据
IOUtils.readFully(fis,buf,0,buf.length);
value.set(buf,0,buf.length);
}finally {
IOUtils.closeStream(fis);
}
isProcess = true;
return true;
}
//如果没下一个文件就返回false,有就返回true
return false;
}
@Override
public NullWritable getCurrentKey() throws IOException, InterruptedException {
return NullWritable.get();
}
@Override
public BytesWritable getCurrentValue() throws IOException, InterruptedException {
return value;
}
//是否正在读取数据 不用管
@Override
public float getProgress() throws IOException, InterruptedException {
return 0;
}
@Override
public void close() throws IOException {
}
}
mapper
package com.buba.mapreduce.wholefile;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import java.io.IOException;
public class WholeMapper extends Mapper<NullWritable, BytesWritable, Text,BytesWritable> {
Text k = new Text();
@Override
protected void setup(Context context) throws IOException, InterruptedException {
//获取切片路径
FileSplit fileSplit = (FileSplit)context.getInputSplit();
Path path = fileSplit.getPath();
//设置key
k.set(path.toString());
}
@Override
protected void map(NullWritable key, BytesWritable value, Context context) throws IOException, InterruptedException {
//控制输出数据
context.write(k,value);
}
}
reducer
package com.buba.mapreduce.wholefile;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
import java.io.IOException;
import java.util.Iterator;
public class WholeReducer extends Reducer<Text, BytesWritable,Text,Text> {
@Override
protected void reduce(Text key, Iterable<BytesWritable> values, Context context) throws IOException, InterruptedException {
Iterator<BytesWritable> iterator = values.iterator();
while (iterator.hasNext()){
BytesWritable next = iterator.next();
byte[] bytes = next.getBytes();
String s = new String(bytes);
context.write(key,new Text(s));
}
}
}
driver
package com.buba.mapreduce.wholefile;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
public class WholeDriver {
public static void main(String[] args)throws Exception {
//1.获取job信息
Configuration configuration = new Configuration();
Job job = Job.getInstance(configuration);
//2.获取jar的存储路径
job.setJarByClass(WholeDriver.class);
job.setInputFormatClass(WholeFileInputformat.class);
//设置输出文件的格式为sequencefile
job.setOutputFormatClass(SequenceFileOutputFormat.class);
//3.关联map和reduce的class类
job.setMapperClass(WholeMapper.class);
//job.setReducerClass(TableReducer.class);
//4.设置map阶段输出key和value类型
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(BytesWritable.class);
//5.设置最后输入数据的key和value的类型
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(BytesWritable.class);
//6.设置输入数据的路径和输出数据的路径
FileInputFormat.setInputPaths(job,new Path(args[0]));
FileOutputFormat.setOutputPath(job,new Path(args[1]));
//7.提交
boolean b = job.waitForCompletion(true);
System.exit(b?0:1);
}
}