无论hdfs还是mapreduce,对于小文件都有损效率,实践中,又难免面临处理大量小文件的场
景,此时,就需要有相应解决方案
1.2 分析
小文件的优化无非以下几种方式:
1、 在数据采集的时候,就将小文件或小批数据合成大文件再上传HDFS
2、 在业务处理之前,在HDFS上使用mapreduce程序对小文件进行合并
3、 在mapreduce处理时,可采用combineInputFormat提高效率
1.3 实现
本节实现的是上述第二种方式
程序的核心机制:
自定义一个InputFormat
改写RecordReader,实现一次读取一个完整文件封装为KV
在输出时使用SequenceFileOutPutFormat输出合并文件
代码如下
自定义InputFormat:
package com.hadoop.inputformat;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import java.io.IOException;
public class MyInputFormat extends FileInputFormat<NullWritable, BytesWritable> {
@Override
public RecordReader<NullWritable, BytesWritable> createRecordReader(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
//1. 创建自定义RecordReader对象
MyRecordReader myRecordReader = new MyRecordReader();
//2. 将 InputSplit inputSplit, TaskAttemptContext taskAttemptContext 传给 RecordReader
myRecordReader.initialize(inputSplit,taskAttemptContext);
return myRecordReader;
}
/**
*
* 设置文件是否可以被切割
**/
@Override
protected boolean isSplitable(JobContext context, Path filename) {
return false;
}
}
自定义RecordReader:
package com.hadoop.inputformat;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import java.io.IOException;
public class MyRecordReader extends RecordReader<NullWritable, BytesWritable> {
private Configuration configuration = null;
private FileSplit fileSplit = null;
// 定义是否读取完文件标识符
private boolean processed = false;
private BytesWritable bytesWritable ;
private FileSystem fileSystem = null;
private FSDataInputStream inputStream =null;
// 进行初始化工作
@Override
public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
// 获取文件切片
fileSplit = (FileSplit) inputSplit;
// 获取 configuration对象
configuration = taskAttemptContext.getConfiguration();
}
// 该方法用户获取K1 和V1
@Override
public boolean nextKeyValue() throws IOException, InterruptedException {
if(!processed){
// 1. 获取源文件的字节输入流
// 获取源文件的文件系统FileSystem
fileSystem = FileSystem.get(configuration);
inputStream = fileSystem.open(fileSplit.getPath());
// 2. 获取源文件数据到普通的字节数组 (byte[])
byte[] bytes = new byte[(int) fileSplit.getLength()];
IOUtils.readFully(inputStream, bytes, 0, (int) fileSplit.getLength());
// 3.将字节数组中的数据封装到bytesWritable 得到V1
bytesWritable = new BytesWritable();
bytesWritable.set(bytes,0,(int) fileSplit.getLength());
processed = true;
return true;
}
return false;
}
// 返回K1
@Override
public NullWritable getCurrentKey() throws IOException, InterruptedException {
return NullWritable.get();
}
// 返回V1
@Override
public BytesWritable getCurrentValue() throws IOException, InterruptedException {
return bytesWritable;
}
// 获取文件读取进度
@Override
public float getProgress() throws IOException, InterruptedException {
return 0;
}
// 释放资源
@Override
public void close() throws IOException {
inputStream.close();
fileSystem.close();
}
}
Mapper类:
package com.hadoop.inputformat;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import java.io.IOException;
public class SequenceFileMapper extends Mapper<NullWritable, BytesWritable, Text,BytesWritable> {
@Override
protected void map(NullWritable key, BytesWritable value, Context context) throws IOException, InterruptedException {
// 1. 获取文件的名字,作为K2
FileSplit fileSplit = (FileSplit)context.getInputSplit();
String fileName = fileSplit.getPath().getName();
// 2. 将K2和 V2 写入上下文中
context.write(new Text(fileName),value);
}
}
主类:
package com.hadoop.inputformat;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
public class JobMain extends Configured implements Tool {
@Override
public int run(String[] strings) throws Exception {
//1.获取Job对象
Job job = Job.getInstance(super.getConf(), "job_MyinputFormat");
//2. 设置job任务
// 第一步: 设置输入类和输入路径
job.setInputFormatClass(MyInputFormat.class);
MyInputFormat.addInputPath(job,new Path("file:///E:\\input\\myinputformat"));
// 第二步: 设置Mapper类和数据类型
job.setMapperClass(SequenceFileMapper.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(BytesWritable.class);
// 第七部: 不需要设置 reducer 但是必须设置数据类型
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(BytesWritable.class);
// 设置输出类
job.setOutputFormatClass(SequenceFileOutputFormat.class);
SequenceFileOutputFormat.setOutputPath(job, new Path("file:///E:\\out\\myinputformat"));
// 3. 等到job任务执行结束
boolean b = job.waitForCompletion(true);
return b ? 0 : 1;
}
public static void main(String[] args) throws Exception {
Configuration configuration = new Configuration();
int run = ToolRunner.run(configuration, new JobMain(), args);
System.exit(run);
}
}
自定义outputFormat:
2.1 需求
现在有一些订单的评论数据,需求,将订单的好评与差评进行区分开来,将最终的数据分开
到不同的文件夹下面去,数据内容参见资料文件夹,其中数据第九个字段表示好评,中评,
差评。0:好评,1:中评,2:差评
2.2 分析
程序的关键点是要在一个mapreduce程序中根据数据的不同输出两类结果到不同目录,这类灵
活的输出需求可以通过自定义outputformat来实现
2.3 实现
实现要点:
1、 在mapreduce中访问外部资源
2、 自定义outputformat,改写其中的recordwriter,改写具体输出数据的方法write()
第一步:自定义MyOutputFormat:
package com.hadoop.outputformat;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import java.io.IOException;
public class MyOutputFormat extends FileOutputFormat<Text, NullWritable> {
@Override
public RecordWriter<Text, NullWritable> getRecordWriter(TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
// 1. 获取目标文件的输出流(两个)
FileSystem fileSystem = FileSystem.get(taskAttemptContext.getConfiguration());
FSDataOutputStream badCommentsOutputStream = fileSystem.create(new Path("file:///E:\\out\\bad_comment\\bad_comment.txt"));
FSDataOutputStream goodCommentsOutputStream = fileSystem.create(new Path("file:///E:\\out\\good_comment\\good_comment.txt"));
// 2. 将输出流传给MyRecordWriter
MyRecordWriter myRecordWriter = new MyRecordWriter(goodCommentsOutputStream, badCommentsOutputStream);
return myRecordWriter;
}
}
第二步:自定义Mapper类
package com.hadoop.outputformat;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import java.io.IOException;
public class MyOutputFrmatMapper extends Mapper<LongWritable, Text,Text, NullWritable> {
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
context.write(value,NullWritable.get());
}
}
第三步:主类JobMain
package com.hadoop.outputformat;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import java.io.IOException;
public class MyRecordWriter extends RecordWriter<Text, NullWritable> {
private FSDataOutputStream badCommentsOutputStream ;
private FSDataOutputStream goodCommentsOutputStream ;
public MyRecordWriter() {
}
public MyRecordWriter(FSDataOutputStream badCommentsOutputStream, FSDataOutputStream goodCommentsOutputStream) {
this.badCommentsOutputStream = badCommentsOutputStream;
this.goodCommentsOutputStream = goodCommentsOutputStream;
}
/**
* 行文本内容
*
**/
@Override
public void write(Text text, NullWritable nullWritable) throws IOException, InterruptedException {
// 1. 从行文本数据 获取第9个字段
String[] split = text.toString().split("\t");
String numStr = split[9];
// 2. 根据字段的值,判断评论的类型,然后将对应的数据写入不同的文件夹文件中
if(Integer.parseInt(numStr) <=1){
// 好评
goodCommentsOutputStream.write(text.toString().getBytes());
goodCommentsOutputStream.write("\r\n".getBytes());
}else{
// 差评
badCommentsOutputStream.write(text.toString().getBytes());
badCommentsOutputStream.write("\r\n".getBytes());
}
}
@Override
public void close(TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
IOUtils.closeStream(goodCommentsOutputStream);
IOUtils.closeStream(badCommentsOutputStream);
}
}