【MapReduce】使用MapReduce来实现数据清洗

• 数据分析
  
  一共有13个字段，分别是出发地、目的价格、节省、路线名、酒店、房间、去程航司、去程方式、去程时间、回程航司、回程方式、回程时间
  数据下载地址: 下载
• 需求
  删除含有空值的数据
  删除重复的数据
  我们假设价格在1000-3500之间为合理值，去除价格异常的数据
  节省>价格为异常，去除节省异常值
  酒店只保留名称，其他的多余信息删除

代码实现

自定义类

import org.apache.hadoop.io.WritableComparable;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;

public class Map_Trip implements WritableComparable<Map_Trip> {
    private String start_loc;
    private String end_loc;
    private String price;
    private String save;
    private String line;
    private String hotel;
    private String room;
    private String start_company;
    private String start_type;
    private String start_time;
    private String end_comany;
    private String end_type;
    private String end_time;

    @Override
    public int compareTo(Map_Trip o) {
        return 0;
    }

    @Override
    public void write(DataOutput dataOutput) throws IOException {
        dataOutput.writeUTF(start_loc);
        dataOutput.writeUTF(end_loc);
        dataOutput.writeUTF(price);
        dataOutput.writeUTF(save);
        dataOutput.writeUTF(line);
        dataOutput.writeUTF(hotel);
        dataOutput.writeUTF(room);
        dataOutput.writeUTF(start_company);
        dataOutput.writeUTF(start_type);
        dataOutput.writeUTF(start_time);
        dataOutput.writeUTF(end_comany);
        dataOutput.writeUTF(end_type);
        dataOutput.writeUTF(end_time);
    }

    @Override
    public void readFields(DataInput dataInput) throws IOException {
        start_loc = dataInput.readUTF();
        end_loc = dataInput.readUTF();
        price = dataInput.readUTF();
        save = dataInput.readUTF();
        line = dataInput.readUTF();
        hotel = dataInput.readUTF();
        room = dataInput.readUTF();
        start_company = dataInput.readUTF();
        start_type = dataInput.readUTF();
        start_time = dataInput.readUTF();
        end_comany = dataInput.readUTF();
        end_type = dataInput.readUTF();
        end_time = dataInput.readUTF();
    }

    public void set(String start_loc, String end_loc, String price, String save, String line, String hotel, String room, String start_company, String start_type, String start_time, String end_comany, String end_type, String end_time) {
        this.start_loc = start_loc;
        this.end_loc = end_loc;
        this.price = price;
        this.save = save;
        this.line = line;
        this.hotel = hotel;
        this.room = room;
        this.start_company = start_company;
        this.start_type = start_type;
        this.start_time = start_time;
        this.end_comany = end_comany;
        this.end_type = end_type;
        this.end_time = end_time;
    }

    @Override
    public String toString() {
        return start_loc + "\t" +
                end_loc + "\t" +
                price + "\t" +
                save + "\t" +
                line + "\t" +
                hotel + "\t" +
                room + "\t" +
                start_company + "\t" +
                start_type + "\t" +
                start_time + "\t" +
                end_comany + "\t" +
                end_type + "\t" +
                end_time;
    }

    public String getStart_loc() {
        return start_loc;
    }

    public void setStart_loc(String start_loc) {
        this.start_loc = start_loc;
    }

    public String getEnd_loc() {
        return end_loc;
    }

    public void setEnd_loc(String end_loc) {
        this.end_loc = end_loc;
    }

    public String getPrice() {
        return price;
    }

    public void setPrice(String price) {
        this.price = price;
    }

    public String getSave() {
        return save;
    }

    public void setSave(String save) {
        this.save = save;
    }

    public String getLine() {
        return line;
    }

    public void setLine(String line) {
        this.line = line;
    }

    public String getHotel() {
        return hotel;
    }

    public void setHotel(String hotel) {
        this.hotel = hotel;
    }

    public String getRoom() {
        return room;
    }

    public void setRoom(String room) {
        this.room = room;
    }

    public String getStart_company() {
        return start_company;
    }

    public void setStart_company(String start_company) {
        this.start_company = start_company;
    }

    public String getStart_type() {
        return start_type;
    }

    public void setStart_type(String start_type) {
        this.start_type = start_type;
    }

    public String getStart_time() {
        return start_time;
    }

    public void setStart_time(String start_time) {
        this.start_time = start_time;
    }

    public String getEnd_comany() {
        return end_comany;
    }

    public void setEnd_comany(String end_comany) {
        this.end_comany = end_comany;
    }

    public String getEnd_type() {
        return end_type;
    }

    public void setEnd_type(String end_type) {
        this.end_type = end_type;
    }

    public String getEnd_time() {
        return end_time;
    }

    public void setEnd_time(String end_time) {
        this.end_time = end_time;
    }
}

Mapper阶段

import com.alibaba.fastjson.JSONObject;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;

public class MapTest extends Mapper<LongWritable, Text, Map_Trip, NullWritable> {
    Map_Trip k = new Map_Trip();

    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        JSONObject jsonObject = JSONObject.parseObject(value.toString());
        String[] datas = new String[13];
        datas[0] = jsonObject.getString("start_loc");
        datas[1] = jsonObject.getString("end_loc");
        datas[2] = jsonObject.getString("price");
        datas[3] = jsonObject.getString("save");
        datas[4] = jsonObject.getString("line");
        datas[5] = jsonObject.getString("hotel");
        datas[6] = jsonObject.getString("room");
        datas[7] = jsonObject.getString("start_company");
        datas[8] = jsonObject.getString("start_type");
        datas[9] = jsonObject.getString("start_time");
        datas[10] = jsonObject.getString("end_comany");
        datas[11] = jsonObject.getString("end_type");
        datas[12] = jsonObject.getString("end_time");
        //去空
        for (String data : datas) {
            if (data.equals("") || data == null) {
                return;
            }
        }
        //去除价格异常值
        if (Integer.parseInt(datas[2]) < 1000 || Integer.parseInt(datas[2]) > 3500) {
            return;
        }
        //去除节省异常值
        if (Integer.parseInt(datas[2]) <= Integer.parseInt(datas[3])) {
            return;
        }
        //截取酒店名称
        datas[5] = datas[5].split(" ")[0];
        //输出
        k.set(datas[0], datas[1], datas[2], datas[3], datas[4], datas[5], datas[6], datas[7], datas[8], datas[9], datas[10], datas[11], datas[12]);
        context.write(k, NullWritable.get());
    }
}

Group 阶段

import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableComparator;

public class Group extends WritableComparator {
    @Override
    public int compare(WritableComparable a, WritableComparable b) {
        Map_Trip trip1 = (Map_Trip)a;
        Map_Trip trip2 = (Map_Trip)b;
        return trip1.toString().compareTo(trip2.toString());
    }
    public Group(){
        super(Map_Trip.class,true);
    }
}

Reduce阶段

import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;

public class RedTest extends Reducer<Map_Trip, NullWritable,Map_Trip,NullWritable> {
    @Override
    protected void reduce(Map_Trip key, Iterable<NullWritable> values, Context context) throws IOException, InterruptedException {
        context.write(key,NullWritable.get());
    }
}

Driver阶段

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import java.io.File;

public class DriTest {
    public static void main(String[] args) throws Exception {
        File file = new File("D:\\MP\\Trip\\output");
        if (file.exists()) {
            delFile(file);
            driver();
        } else {
            driver();
        }
    }

    public static void delFile(File file) {
        File[] files = file.listFiles();
        if (files != null && files.length != 0) {
            for (int i = 0; i < files.length; i++) {
                delFile(files[i]);
            }
        }
        file.delete();
    }

    public static void driver() throws Exception {
        Configuration conf = new Configuration();
//        conf.set("fs.default","hdfs://192.168.0.155:9000");
        Job job = Job.getInstance(conf);

        job.setJarByClass(DriTest.class);
        job.setMapperClass(MapTest.class);
        job.setReducerClass(RedTest.class);

        job.setMapOutputKeyClass(Map_Trip.class);
        job.setMapOutputValueClass(NullWritable.class);
        job.setOutputKeyClass(Map_Trip.class);
        job.setOutputValueClass(NullWritable.class);

        job.setGroupingComparatorClass(Group.class);

        FileInputFormat.setInputPaths(job, "D:\\MP\\Trip\\input");
        FileOutputFormat.setOutputPath(job, new Path("D:\\MP\\Trip\\output"));
        boolean b = job.waitForCompletion(true);
        System.exit(b ? 0 : 1);
    }
}

处理后的数据