MapReduce编程场景之数据分类输出

MapReduce编程场景之数据分类输出

（一）需求
现有一些原始日志需要做增强解析处理，流程：
1、 从原始日志文件中读取数据
2、 根据业务获取业务数据库的数据
3、 根据某个连接条件获取相应的连接结果

（二）分析
程序的关键点是要在一个 MapReduce 程序中根据数据的不同输出两类结果到不同目录，这
类灵活的输出需求可以通过自定义 OutputFormat 来实现
（三）实现
实现要点：
1、 在 MapReduce 中访问外部资源
2、 自定义 OutputFormat，改写其中的 RecordWriter，改写具体输出数据的方法 write()以 Score.txt 的 32 条学生考试记录为例，现要求把参考次数>=7 的输出到一个文件/output/out1，然后剩下的不合格的参考输出到另外一个文件/output/out2

/**
*第一步，实现自己的 OutputFormat
*@author lv_hulk
*/
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
public class LongOutputFormat extends FileOutputFormat<Text, NullWritable>{
@Override
public RecordWriter<Text, NullWritable> getRecordWriter(
TaskAttemptContext job) throws IOException, InterruptedException {
Configuration configuration = job.getConfiguration();
FileSystem fs = FileSystem.get(configuration);
Path p1 = new Path("d:/outputformat/out1");
Path p2 = new Path("d:/outputformat/out2");
FSDataOutputStream out1 = fs.create(p1);
FSDataOutputStream out2 = fs.create(p2);
return new MyRecordWriter(out1, out2);
}
static class MyRecordWriter extends RecordWriter<Text, NullWritable>{
FSDataOutputStream fsdout = null;
FSDataOutputStream fsdout1 = null;
public MyRecordWriter(FSDataOutputStream fsdout,
FSDataOutputStream fsdout1) {
super();
this.fsdout = fsdout;
this.fsdout1 = fsdout1;
}
@Override
public void write(Text key, NullWritable value) throws IOException,
InterruptedException {
String[] strs = key.toString().split("::");
if(strs[0].equals("1")){
fsdout.write((strs[1]+"\n").getBytes());
}else{
fsdout1.write((strs[1]+"\n").getBytes());
}
}
@Override
public void close(TaskAttemptContext context) throws IOException,
InterruptedException {
IOUtils.closeStream(fsdout);
IOUtils.closeStream(fsdout1);
}
}
}

/**
*第二步，实现 MapReduce 程序
*@author lv_hulk
*/
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
public class MultipleOutputMR {
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
Job job = Job.getInstance(conf);
job.setJarByClass(MultipleOutputMR.class);
job.setMapperClass(MultipleOutputMRMapper.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(NullWritable.class);
job.setOutputFormatClass(LongOutputFormat.class);
FileInputFormat.setInputPaths(job, "d:/score/input");
Path outPath = new Path("d:/score/output_success");
FileSystem fs = FileSystem.get(conf);
if (fs.exists(outPath)) {
fs.delete(outPath, true);
}
FileOutputFormat.setOutputPath(job, outPath);
job.waitForCompletion(true);
}
static class MultipleOutputMRMapper extends
Mapper<LongWritable, Text, Text, NullWritable> {
@Override
protected void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
// value
// 参考次数大于 7 次算合格
String[] splits = value.toString().split("\t");
if (splits.length > 9) {
context.write(new Text("1::" + value.toString()), NullWritable.get());
} else {
context.write(new Text("2::" + value.toString()), NullWritable.get());
}
}
}
}