hadoop_MapReduce 自定义outputFormat

适用场景

将最终的数据分开到不同的文件夹下面去

分析

程序的关键点是要在一个mapreduce程序中根据数据的不同，输出两类结果到不同目录，这类灵活的输出需求可以通过自定义outputformat来实现

实现思路

实现要点：

• 1、在mapreduce中访问外部资源
• 2、自定义outputformat，改写其中的recordwriter，改写具体输出数据的方法write()

代码实现：

自定义一个outputformat

import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;

public class DiyOutputFormat extends FileOutputFormat<Text, NullWritable> {
    @Override
    public RecordWriter<Text, NullWritable> getRecordWriter(TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
        FileSystem fileSystem = FileSystem.get(taskAttemptContext.getConfiguration());
        FSDataOutputStream goodReview = fileSystem.create(new Path("输出数据路径"));
        FSDataOutputStream bedReview = fileSystem.create(new Path("输出数据路径"));
        DiyRecordWriter diyRecordWriter = new DiyRecordWriter(goodReview,bedReview);
        return diyRecordWriter;
    }
}

定义RecordWriter类

import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;

import java.io.IOException;

public class DiyRecordWriter extends RecordWriter<Text, NullWritable> {
    private FSDataOutputStream goodReview;
    private FSDataOutputStream bedReview;

    public DiyRecordWriter(FSDataOutputStream goodReview, FSDataOutputStream bedReview) {
        this.goodReview = goodReview;
        this.bedReview = bedReview;

    }

	//根据项目需求编写逻辑
    @Override
    public void write(Text text, NullWritable nullWritable) throws IOException, InterruptedException {
        String[] split = text.toString().trim().split("\\t");
        if (split[9].equals("0")) {
            goodReview.write(text.toString().getBytes());
            goodReview.write("\r\n".getBytes());
        } else {
            bedReview.write(text.toString().getBytes());
            bedReview.write("\r\n".getBytes());
        }

    }

    @Override
    public void close(TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
        if (goodReview != null) {
            goodReview.close();
        }
        if (bedReview != null) {
            bedReview.close();
        }
    }
}

定义Map

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;

public class DiyMap extends Mapper<LongWritable, Text, Text, NullWritable> {
    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        context.write(value, NullWritable.get());

    }
}

定义Driver主类

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;

public class DiyDriver {
    public static void main(String[] args) throws Exception {

        Job job = Job.getInstance(new Configuration(), "");

        job.setJarByClass(DiyDriver.class);
        job.setMapperClass(DiyMap.class);

        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(NullWritable.class);

        job.setInputFormatClass(TextInputFormat.class);
        TextInputFormat.addInputPath(job, new Path("读取数据路径"));

        job.setOutputFormatClass(DiyOutputFormat.class);
        DiyOutputFormat.setOutputPath(job, new Path("输出的success数据路径"));

        boolean b = job.waitForCompletion(true);
        System.exit(b ? 0 : 1);
    }
}

看了不点赞，坤坤咬你蛋！！！