MapReduce - Reduce Join 案例

Reduce Join工作原理

Reduce Join案例实操

2．需求分析
通过将关联条件作为Map输出的key，将两表满足Join条件的数据并携带数据所来源的文件信息，发往同一个ReduceTask，在Reduce中进行数据的串联

相关代码

TableBean.java

package MapReduceJoin;

import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparator;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;

public class TableBean implements Writable,Comparable<TableBean> {
    private String orderId = new String();  //订单编号
    private String pid = new String();  //品牌编号
    private int amount = 0;  //数量
    private String pname = new String();  //品牌名称

    public String getFlag() {
        return flag;
    }

    public void setFlag(String flag) {
        this.flag = flag;
    }

    private String flag = new String();  //标志，数据来源

    public String getOrderId() {
        return orderId;
    }

    public void setOrderId(String orderId) {
        this.orderId = orderId;
    }

    public String getPid() {
        return pid;
    }

    public void setPid(String pid) {
        this.pid = pid;
    }

    public int getAmount() {
        return amount;
    }

    public void setAmount(int amount) {
        this.amount = amount;
    }

    public String getPname() {
        return pname;
    }

    public void setPname(String pname) {
        this.pname = pname;
    }

    @Override
    public void write(DataOutput out) throws IOException {
        out.writeUTF(orderId);
        out.writeUTF(pid);
        out.writeInt(amount);
        out.writeUTF(pname);
        out.writeUTF(flag);
    }

    @Override
    public void readFields(DataInput in) throws IOException {
        this.orderId = in.readUTF();
        this.pid = in.readUTF();
        this.amount = in.readInt();
        this.pname = in.readUTF();
        this.flag = in.readUTF();
    }

    @Override
    public String toString() {
        return orderId + '\t' + pname + '\t' + amount;
    }

    @Override
    public int compareTo(TableBean o) {
        return this.pid.compareTo(o.getPid());
    }
}

TableMapper.java

package MapReduceJoin;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;

import java.io.IOException;

public class TableMapper extends Mapper<LongWritable, Text,Text, TableBean> {
    Text k = new Text();  //编号
    TableBean tb = new TableBean();

    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        FileSplit fs = (FileSplit) context.getInputSplit();
        String fileName = fs.getPath().getName();
        String[] fields = value.toString().split("\t");

        if(fileName.equals("order.txt")){
            tb.setOrderId(fields[0]);
            tb.setPid(fields[1]);
            tb.setAmount(Integer.parseInt(fields[2]));
            tb.setPname("");
            tb.setFlag("Order");
        }else if(fileName.equals("pd.txt")){
            tb.setOrderId("");
            tb.setPid(fields[0]);
            tb.setAmount(0);
            tb.setPname(fields[1]);
            tb.setFlag("PD");
        }

        k.set(tb.getPid());
        context.write(k,tb);
    }
}

TableReducer.java

package MapReduceJoin;

import org.apache.commons.beanutils.BeanUtils;
import org.apache.commons.lang.ObjectUtils;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
import org.codehaus.jackson.map.util.BeanUtil;

import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.List;

public class TableReducer extends Reducer<Text,TableBean,TableBean,NullWritable> {
    @Override
    protected void reduce(Text key, Iterable<TableBean> values, Context context) throws IOException, InterruptedException {
        //1001	01	1 和 01	小米 结合输出为 1001 小米 数量

        List<TableBean> list = new ArrayList<>();
        String pname = "";

        for(TableBean tb : values){
            if (tb.getFlag().equals("Order")){
                TableBean newTb = new TableBean();

                try {
                    BeanUtils.copyProperties(newTb,tb);
                    list.add(newTb);
                } catch (IllegalAccessException e) {
                    e.printStackTrace();
                } catch (InvocationTargetException e) {
                    e.printStackTrace();
                }
            }else{
                if(pname.isEmpty()){
                    pname = tb.getPname();
                }
            }
        }

        for(TableBean tb : list){
            tb.setPname(pname);
            context.write(tb,NullWritable.get());
        }
    }
}

结果

坑

在reduce函数中，如果直接list.add(tb);那么结果是不正确的。因为Iterable<TableBean> values迭代器遍历的时候用tb赋值只是改变地址值，并不会真的把对象拷贝给你。所以使用BeanUtils.copyProperties(newTb,tb);