MapReduce自定义排序WriteComparable

MapReduce自定义排序

MapReduce会按数据的Key进行默认的排序，如果想要进行定制排序，可以实现WritableComparable接口，重写compareTo方法

注意，以上的办法是作用于MapReduce阶段的Key值进行排序，如果要对Value进行排序，则这里的写法不会起作用，按照Value的排序，在下面给出

实现WritableComparable接口之后需要实现三个方法，分别是序列化方法，反序列化方法和compareTo方法

需求，按照时间戳的正序进行排序

要求在AISDataBean类中对其属性unixTime按照从小到大进行排序

AISDataBean类得编写：

package com.gis507.test.AISDataSort;

import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;

/**
 * 1、实现Writable接口
 * 2、属性有：speed，unixTime，lon_d，lat_d，(String)draught(类型),Length,Width,CallSign,Course,Rot
 */
public class AISDataBean implements Writable {

    private double speed;
    private long unixTime;
    private double lon_d;
    private double lat_d;
    private String draught;
    private String length;
    private String width;
    private String callsign;
    private double course;
    private double rot;

    public AISDataBean(double speed, long unixTime, double lon_d, double lat_d, String draught, String length, String width, String callsign, double course, double rot) {
        this.speed = speed;
        this.unixTime = unixTime;
        this.lon_d = lon_d;
        this.lat_d = lat_d;
        this.draught = draught;
        this.length = length;
        this.width = width;
        this.callsign = callsign;
//        this.heading = heading;
        this.course = course;
        this.rot = rot;
    }

    public AISDataBean() {
    }

    public double getSpeed() {
        return speed;
    }

    public void setSpeed(double speed) {
        this.speed = speed;
    }

    public long getUnixTime() {
        return unixTime;
    }

    public void setUnixTime(long unixTime) {
        this.unixTime = unixTime;
    }

    public double getLon_d() {
        return lon_d;
    }

    public void setLon_d(double lon_d) {
        this.lon_d = lon_d;
    }

    public double getLat_d() {
        return lat_d;
    }

    public void setLat_d(double lat_d) {
        this.lat_d = lat_d;
    }

    public String getDraught() {
        return draught;
    }

    public void setDraught(String draught) {
        this.draught = draught;
    }

    public String getLength() {
        return length;
    }

    public void setLength(String length) {
        this.length = length;
    }

    public String getWidth() {
        return width;
    }

    public void setWidth(String width) {
        this.width = width;
    }

    public String getCallsign() {
        return callsign;
    }

    public void setCallsign(String callsign) {
        this.callsign = callsign;
    }

    public double getCourse() {
        return course;
    }

    public void setCourse(double course) {
        this.course = course;
    }

    public double getRot() {
        return rot;
    }

    public void setRot(double rot) {
        this.rot = rot;
    }

    @Override
    public String toString() {
        return speed + "," + unixTime + "," + lon_d + "," + lat_d + "," + draught + ","
                + length + "," + width + "," + callsign + "," + course + "," + rot ;
    }


    @Override
    public void write(DataOutput dataOutput) throws IOException {

        //speed，unixTime，lon_d，lat_d，(String)draught(类型),Length,Width,CallSign,Course,Rot
        dataOutput.writeDouble(speed);
        dataOutput.writeLong(unixTime);
        dataOutput.writeDouble(lon_d);
        dataOutput.writeDouble(lat_d);
        dataOutput.writeUTF(draught);
        dataOutput.writeUTF(length);
        dataOutput.writeUTF(width);
        dataOutput.writeUTF(callsign);
        dataOutput.writeDouble(course);
        dataOutput.writeDouble(rot);

    }

    @Override
    public void readFields(DataInput dataInput) throws IOException {

        this.speed = dataInput.readDouble();
        this.unixTime = dataInput.readLong();
        this.lon_d = dataInput.readDouble();
        this.lat_d = dataInput.readDouble();
        this.draught = dataInput.readUTF();
        this.length = dataInput.readUTF();
        this.width = dataInput.readUTF();
        this.callsign = dataInput.readUTF();
        this.course = dataInput.readDouble();
        this.rot = dataInput.readDouble();

    }

    @Override
    public int compareTo(AISDataBean o) {

        if (this.getUnixTime() > o.getUnixTime()){
            return 1;
        }else{
            return -1;
        }
    }
}

Mapper和Reducer和Driver就正常按照逻辑写就行

注意，这里的排序需要把AISDataBean当做key进行处理，如果要对value进行排序，通常的方法是：

1、在map阶段将AISDataBean当做是key，在Reducer的输出阶段将key和value互换位置，这里就要注意在Driver中设置输出的key和value类型时设置正确

2、除了上述方法，还可以在reducer阶段对value进行排序

@Override
    protected void reduce(Text key, Iterable<AISDataBean> values, Context context) throws IOException, InterruptedException {

        //实例化一个列表，用于存放数据
        List<AISDataBean> aisList = new ArrayList<>(1000);

        //遍历values
        for (AISDataBean value : values) {

            //将value添加到列表
            aisList.add(new 		AISDataBean(value.getSpeed(),value.getUnixTime(),value.getLon_d(),value.getLat_d(),value.getDraught(),value.getLength(),value.getWidth(),value.getCallsign(),value.getCourse(),value.getRot()));   
            
            log.info(key + "," + value.getUnixTime());
        }

        //使用lambda表达式对aisList按照时间进行排序
//        aisList.stream().sorted(Comparator.comparingLong(AISDataBean::getUnixTime));

        //出了上处的写法外，还可以使用lambda表达式这样排序
        aisList.sort((AISDataBean a1,AISDataBean a2) -> a1.compareTo(a2));
		//遍历列表
        for (AISDataBean aisDataBean : aisList) {
			//写出
            context.write(key,aisDataBean);
        }
    }

注意：在使用的compareTo是在Bean中定义之后使用的，否则就会使用java提供的默认compareTo方法