MR自定义排序

MR会在shuffle阶段根据数据的key值进行排序，我们可以自定义类实现WritableComparable接口来指定排序规则。

代码实现：

1.自定义排序类

package com.aura.hadoop.compare;


import org.apache.hadoop.io.WritableComparable;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;

/**
 * @author panghu
 * @description 自定义排序
 * @create 2021-02-14-16:49
 */
public class FlowBeanCompare implements WritableComparable<FlowBeanCompare> {
    private Long upFlow;
    private Long downFlow;
    private Long sumFlow;

    public void set(Long upFlow, Long downFlow) {
        this.upFlow = upFlow;
        this.downFlow = downFlow;
        this.sumFlow = upFlow + downFlow;
    }

    public Long getUpFlow() {
        return upFlow;
    }

    public void setUpFlow(Long upFlow) {
        this.upFlow = upFlow;
    }

    public Long getDownFlow() {
        return downFlow;
    }

    public void setDownFlow(Long downFlow) {
        this.downFlow = downFlow;
    }

    public Long getSumFlow() {
        return sumFlow;
    }

    public void setSumFlow(Long sumFlow) {
        this.sumFlow = sumFlow;
    }

    @Override
    public String toString() {
        return upFlow + "\t" + downFlow + "\t" + sumFlow;
    }

    /**
     * 把要序列化的对象的属性发送给框架
     *
     * @throws IOException
     */
    @Override
    public void write(DataOutput dataOutput) throws IOException {
        dataOutput.writeLong(upFlow);
        dataOutput.writeLong(downFlow);
        dataOutput.writeLong(sumFlow);
    }

    /**
     * 填充序列化的对象属性，读写顺序要一致
     *
     * @param dataInput
     * @throws IOException
     */
    @Override
    public void readFields(DataInput dataInput) throws IOException {
        this.upFlow = dataInput.readLong();
        this.downFlow = dataInput.readLong();
        this.sumFlow = dataInput.readLong();
    }

    /**
     * 总流量倒序
     *
     * @param o
     * @return
     */
    @Override
    public int compareTo(FlowBeanCompare o) {
        return Long.compare(o.getSumFlow(), this.getSumFlow());
    }
}

2.Mapper类

package com.aura.hadoop.compare;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;

/**
 * @author panghu
 * @description 利用shuffle阶段的排序，把排序规则所在的类放在map端输出的key上
 * @create 2021-02-15-17:10
 */
public class CompareMapper extends Mapper<LongWritable, Text, FlowBeanCompare, Text>{
    private FlowBeanCompare fb = new FlowBeanCompare();
    private Text phone = new Text();

    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        String line = value.toString();
        String[] split = line.split("\t");

        fb.set(Long.parseLong(split[1]),Long.parseLong(split[2]));
        phone.set(split[0]);

        context.write(fb,phone);
    }
}

3.Reducer类

package com.aura.hadoop.compare;


import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;

/**
 * @author panghu
 * @description
 * @create 2021-02-15-17:19
 */
public class CompareReducer extends Reducer<FlowBeanCompare, Text, Text, FlowBeanCompare> {
    @Override
    protected void reduce(FlowBeanCompare key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
        for (Text value : values) {
            context.write(value,key);
        }
    }
}

4.Driver类

package com.aura.hadoop.compare;


import com.aura.hadoop.flow.bean.FlowBean;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;

/**
 * @author panghu
 * @description
 * @create 2021-02-14-16:49
 */
public class CompareDriver {
    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
        Job job = Job.getInstance(new Configuration());

        job.setJarByClass(CompareDriver.class);

        job.setMapperClass(CompareMapper.class);
        job.setReducerClass(CompareReducer.class);

        job.setMapOutputKeyClass(FlowBeanCompare.class);
        job.setMapOutputValueClass(Text.class);

        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(FlowBeanCompare.class);

        FileInputFormat.setInputPaths(job,new Path("D:\\data\\out\\flow_out"));
        FileOutputFormat.setOutputPath(job,new Path("D:\\data\\out\\compare_out"));

        boolean wait = job.waitForCompletion(true);

        System.exit(wait ? 0 : 1);
    }
}