五十三、Mapreduce之自定义outputformat案例

【一线大厂Java面试题解析+后端开发学习笔记+最新架构讲解视频+实战项目源码讲义】

**开源地址：https://docs.qq.com/doc/DSmxTbFJ1cmN1R2dB **

---

程序主体架构：

（1）LogMapper编写

package org.example.mapreduce.outputformat;



import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.NullWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Mapper;



import java.io.IOException;



public class LogMapper extends Mapper<LongWritable, Text, Text, NullWritable> {



    @Override

    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {

        context.write(value, NullWritable.get());

    }



}

（2）LogReducer编写

package org.example.mapreduce.outputformat;



import org.apache.hadoop.io.NullWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Reducer;



import java.io.IOException;



public class LogReducer extends Reducer<Text, NullWritable, Text, NullWritable> {

    @Override

    protected void reduce(Text key, Iterable<NullWritable> values, Context context) throws IOException, InterruptedException {



        //为防止输入相同网址输出一条数据，进行循环迭代

        for (NullWritable value : values) {

            context.write(key, NullWritable.get());

        }

    }

}

（3）LogOutputFormat编写

package org.example.mapreduce.outputformat;





import org.apache.hadoop.io.NullWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.RecordWriter;

import org.apache.hadoop.mapreduce.TaskAttemptContext;

import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;



import java.io.IOException;



public class LogOutputFormat extends FileOutputFormat<Text, NullWritable> {



    @Override

    public RecordWriter<Text, NullWritable> getRecordWriter(TaskAttemptContext job) throws IOException, InterruptedException {

        LogRecordWriter lrw = new LogRecordWriter(job);



        return lrw;

    }

}

（4）LogRecordWriter编写

package org.example.mapreduce.outputformat;



import org.apache.hadoop.fs.FSDataOutputStream;

import org.apache.hadoop.fs.FileSystem;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.IOUtils;

import org.apache.hadoop.io.NullWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.RecordWriter;

import org.apache.hadoop.mapreduce.TaskAttemptContext;



import java.io.IOException;



public class LogRecordWriter extends RecordWriter<Text, NullWritable> {

    private FSDataOutputStream tuomasiOut;

    private FSDataOutputStream otherOut;



    public LogRecordWriter(TaskAttemptContext job) {

        //创建流

        try {

            FileSystem fs = FileSystem.get(job.getConfiguration());

            tuomasiOut = fs.create(new Path("E:\\output\\outputformats\\tuomasi.log"));



            otherOut = fs.create(new Path("E:\\output\\outputformats\\other.log"));

        } catch (IOException e) {

            e.printStackTrace();

        }

    }



    @Override

    public void write(Text key, NullWritable value) throws IOException, InterruptedException {

        String log = key.toString();

        //具体流程

        if (log.contains("tuomasi")) {

            tuomasiOut.writeBytes(log+"\n");

        } else {

            otherOut.writeBytes(log+"\n");

        }

    }



    @Override

    public void close(TaskAttemptContext context) throws IOException, InterruptedException {

        //关闭流

        IOUtils.closeStream(tuomasiOut);

        IOUtils.closeStream(otherOut);

    }

}

（5）LogDriver编写

package org.example.mapreduce.outputformat;





import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.NullWritable;