需求:
在Map,Reduce中,有时需要将一些小文件合并
有3个小文件,现在需要合并成一个文件
1:定义SequenceFileMap类
package inputFormat; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.lib.input.FileSplit; import java.io.IOException; public class SequenceFileMap extends Mapper<NullWritable, BytesWritable, Text, BytesWritable> { Text k = new Text(); @Override protected void setup(Context context) throws IOException, InterruptedException { //1.获取文件切片信息 FileSplit inputSplit = (FileSplit) context.getInputSplit(); //2.获取切片名称 String pathName = inputSplit.getPath().toString(); //3.设置key的输出 k.set(pathName); } @Override protected void map(NullWritable key, BytesWritable value, Context context) throws IOException, InterruptedException { context.write(k, value); } }
2:定义SequenceFileReducer类
package inputFormat; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Reducer; import java.io.IOException; public class SequenceFileReducer extends Reducer<Text, BytesWritable, Text, BytesWritable> { @Override protected void reduce(Text key, Iterable<BytesWritable> values, Context context) throws IOException, InterruptedException { context.write(key, values.iterator().next()); } }
3:定义AllRecordReader类
package inputFormat; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapreduce.InputSplit; import org.apache.hadoop.mapreduce.RecordReader; import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.hadoop.mapreduce.lib.input.FileSplit; import java.io.IOException; public class AllRecordReader extends RecordReader<NullWritable, BytesWritable> { //配置、切片、数据加工、value值 private Configuration configuration; private FileSplit split; private boolean processes = false; private BytesWritable value = new BytesWritable(); public AllRecordReader() { super(); } @Override public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException { split = (FileSplit) inputSplit; configuration = taskAttemptContext.getConfiguration(); } @Override public boolean nextKeyValue() throws IOException, InterruptedException { if (!processes){ //定义缓存区 byte[] contents = new byte[(int) split.getLength()]; //获取文件系统 Path path = split.getPath(); FileSystem fs = path.getFileSystem(configuration); FSDataInputStream fis = fs.open(path); IOUtils.readFully(fis,contents,0,contents.length); value.set(contents,0,contents.length); fis.close(); processes = true; return true; } return false; } @Override public NullWritable getCurrentKey() throws IOException, InterruptedException { return NullWritable.get(); } @Override public BytesWritable getCurrentValue() throws IOException, InterruptedException { return value; } @Override public float getProgress() throws IOException, InterruptedException { return processes ? 1:0; } @Override public void close() throws IOException { } }
4:定义AllFileInputFormat类
package inputFormat; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapreduce.InputSplit; import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.RecordReader; import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import java.io.IOException; public class AllFileInputFormat extends FileInputFormat<NullWritable, BytesWritable> { //是否拆分数据 @Override protected boolean isSplitable(JobContext context, Path filename) { return false; } public RecordReader createRecordReader(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException { AllRecordReader reader = new AllRecordReader(); reader.initialize(inputSplit, taskAttemptContext); return reader; } }
5:设置SequenceFileDriver类
package inputFormat; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat; import java.io.IOException; public class SequenceFileDriver { public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException { args = new String[]{"E:\\bigdata_code\\inputformat","E:\\bigdata_code\\inputformat\\out"}; Configuration conf = new Configuration(); Job job = Job.getInstance(); job.setJarByClass(SequenceFileDriver.class); job.setMapperClass(SequenceFileMap.class); job.setReducerClass(SequenceFileReducer.class); //设置输入的inputFormat job.setInputFormatClass(AllFileInputFormat.class); //设置输出的outputFormat job.setOutputFormatClass(SequenceFileOutputFormat.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(BytesWritable.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(BytesWritable.class); FileInputFormat.setInputPaths(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); job.waitForCompletion(true); } } 输出结果: