MapReduce （8）--MapReduce 自定义outputFormat

1 需求

现在有一些订单的评论数据，需求，将订单的好评与差评进行区分开来，将最终的数据分开到不同的文件夹下面去，数据内容参见资料文件夹，其中数据第九个字段表示好评，中评，差评。0：好评，1：中评，2：差评

2 分析

程序的关键点是要在一个mapreduce程序中根据数据的不同输出两类结果到不同目录，这类灵活的输出需求可以通过自定义outputformat来实现

 

3 实现

实现要点：

1. 在mapreduce中访问外部资源

自定义outputformat，改写其中的recordwriter，改写具体输出数据的方法write()

上代码！！！

第一步：自定义一个outputformat

import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;

/**
 * @author kismet
 * @date 2019-11-19 9:18
 */
public class MyOutputformat  extends FileOutputFormat<Text, NullWritable> {
    @Override
    public RecordWriter<Text, NullWritable> getRecordWriter(TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
        FileSystem fileSystem = FileSystem.get(taskAttemptContext.getConfiguration());
        Path path1 = new Path("E:\\第三学期\\第二阶段\\day22\\5\\自定义outputformat\\outA\\1.txt");
        Path path2 = new Path("E:\\第三学期\\第二阶段\\day22\\5\\自定义outputformat\\outB\\2.txt");
        FSDataOutputStream enhanceOut = fileSystem.create(path1);
        FSDataOutputStream toCrawlOut = fileSystem.create(path2);
        return new MyRecordWriter(enhanceOut,toCrawlOut);
    }
}

 

第二步：自定义一个MyRecordWriter

import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;

import java.io.IOException;

/**
 * @author kismet
 * @date 2019-11-19 9:28
 */
public class MyRecordWriter extends RecordWriter<Text, NullWritable> {
    FSDataOutputStream enhanceOut = null;
    FSDataOutputStream toCrawlOut = null;
    @Override
    public void write(Text key, NullWritable nullWritable) throws IOException, InterruptedException {
        if (key.toString().split("\t")[9].equals("0")){
            toCrawlOut.write(key.toString().getBytes());
            toCrawlOut.write("\r\n".getBytes());
        }else{
            enhanceOut.write(key.toString().getBytes());
            enhanceOut.write("\r\n".getBytes());
        }
    }

    @Override
    public void close(TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
        if(toCrawlOut!=null){
            toCrawlOut.close();
        }
        if(enhanceOut!=null){
            enhanceOut.close();
        }
    }

    public MyRecordWriter(FSDataOutputStream enhanceOut, FSDataOutputStream toCrawlOut) {
        this.enhanceOut = enhanceOut;
        this.toCrawlOut = toCrawlOut;
    }
}

第三步：自定义一个map

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;

/**
 * @author kismet
 */
public class WordCountMap extends Mapper<LongWritable, Text, Text, NullWritable> {

    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        String[] split = value.toString().split("\t");
        String commentStatus = split[9];
        context.write(value,NullWritable.get());
    }
}

第四步：开发mapreduce处理流程

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

/**
 * @author kismet
 * @date 2019-11-19 9:36
 */
public class MyOwnOutputFormatMain extends Configured implements Tool {
    @Override
    public int run(String[] args) throws Exception {
        Job job = Job.getInstance(new Configuration(), "www");
        job.setJarByClass(MyOwnOutputFormatMain.class);
        job.setInputFormatClass(TextInputFormat.class);
        TextInputFormat.addInputPath(job,new Path("E:\\第三学期\\第二阶段\\day22\\5\\自定义outputformat\\input"));
        job.setMapperClass(WordCountMap.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(NullWritable.class);
        job.setOutputFormatClass(MyOutputformat.class);
        MyOutputformat.setOutputPath(job,new Path("E:\\第三学期\\第二阶段\\day22\\5\\自定义outputformat\\outC"));
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(NullWritable.class);
        boolean b = job.waitForCompletion(true);
        return b?0:1;
    }
    public static void main(String[] args) throws Exception {
        Configuration configuration = new Configuration();
        ToolRunner.run(configuration,new MyOwnOutputFormatMain(),args);
    }
}