MapReduce之Join的应用

一、 Reduce Join

实例：

表1 订单数据表t_order

id	pid	amount
1001	01	1
1002	02	2
1003	03	3
1004	01	4
1005	02	5
1006	03	6

表2 商品信息表t_product

pid	pname
01	小米
02	华为
03	格力

将商品信息表中数据根据商品pid合并到订单数据表中。

表3 最终数据形式

id	pname	amount
1001	小米	1
1004	小米	4
1002	华为	2
1005	华为	5
1003	格力	3
1006	格力	6

        通过将关联条件作为Map输出的key，将两表满足Join条件的数据并携带数据所来源的文件信息，发往同一个ReduceTask，在Reduce中进行数据的串联。

（1）创建商品和订单合并后的TableBean类

/**
 * @author LS
 * @date 2021/12/15 10:39
 * @description
 */
public class TableBean  implements Writable {
    private String id; //定义属性值
    private String pid;
    private int amount;
    private String pname;
    private String flag; //用来判断哪个具体的表

    public TableBean() {
    }

    public String getFlag() {
        return flag;
    }

    public void setFlag(String flag) {
        this.flag = flag;
    }


    public String getId() {
        return id;
    }

    public void setId(String id) {
        this.id = id;
    }

    public String getPid() {
        return pid;
    }

    public void setPid(String pid) {
        this.pid = pid;
    }

    public int getAmount() {
        return amount;
    }

    public void setAmount(int amount) {
        this.amount = amount;
    }

    public String getPname() {
        return pname;
    }

    public void setPname(String pname) {
        this.pname = pname;
    }

    @Override
    public String toString() {
        return id+"\t"+pname+"\t"+amount;
    }

    @Override
    public void write(DataOutput dataOutput) throws IOException {
        dataOutput.writeUTF(id);
        dataOutput.writeUTF(pid);
        dataOutput.writeInt(amount);
        dataOutput.writeUTF(pname);
        dataOutput.writeUTF(flag);
    }

    @Override
    public void readFields(DataInput dataInput) throws IOException {
        this.id = dataInput.readUTF();
        this.pid = dataInput.readUTF();
        this.amount = dataInput.readInt();
        this.pname = dataInput.readUTF();
        this.flag = dataInput.readUTF();
    }
}

（2）编写TableMapper类

/**
 * @author LS
 * @date 2021/12/15 10:46
 * @description
 */
public class TableMapper extends Mapper<LongWritable,Text,Text,TableBean> {
    private String filename;
    private Text outK;
    private TableBean outV;
    @Override
    protected void setup(Context context) throws IOException, InterruptedException {
        outK = new Text();
        outV = new TableBean();
        //获取对应文件名称
        InputSplit iSplit = context.getInputSplit();
        FileSplit fileSplit = (FileSplit) iSplit;
        filename = fileSplit.getPath().getName();
    }

    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        String line = value.toString();
            
        //判断是哪个文件,然后针对文件进行不同的操作
        if (filename.contains("order")){
            String[] split = line.split("\t");
            outK.set(split[1]);

            outV.setId(split[0]);
            outV.setPid(split[1]);
            outV.setAmount(Integer.parseInt(split[2]));
            outV.setPname("");
            outV.setFlag("order");
        }else{
            String[] split = line.split("\t");
            outK.set(split[0]);

            outV.setId("");
            outV.setPid(split[0]);
            outV.setAmount(0);
            outV.setPname(split[1]);
            outV.setFlag("pd");
        }
        context.write(outK,outV);
    }
}

（3）编写TableReducer类

/**
 * @author LS
 * @date 2021/12/15 10:57
 * @description
 */
public class TableReducer extends Reducer<Text, TableBean, TableBean, NullWritable> {
    @Override
    protected void reduce(Text key, Iterable<TableBean> values, Context context) throws IOException, InterruptedException {
        ArrayList<TableBean> orderBeans = new ArrayList<>();
        TableBean tableBean = new TableBean();
        
        for (TableBean value : values) {
            //判断数据来自哪个表
            if ("order".equals(value.getFlag())) {
                //创建一个临时TableBean对象接收value
                TableBean tmptableBean = new TableBean();
                try {
                    BeanUtils.copyProperties(tmptableBean, value);
                } catch (IllegalAccessException e) {
                    e.printStackTrace();
                } catch (InvocationTargetException e) {
                    e.printStackTrace();
                }
                //将临时TableBean对象添加到集合orderBeans
                orderBeans.add(tmptableBean);
            } else {
                try {
                    BeanUtils.copyProperties(tableBean, value);
                } catch (IllegalAccessException e) {
                    e.printStackTrace();
                } catch (InvocationTargetException e) {
                    e.printStackTrace();
                }
            }
        }
        //遍历集合orderBeans,替换掉每个orderBean的pid为pname,然后写出
        for (TableBean orderBean : orderBeans) {
            orderBean.setPname(tableBean.getPname());
            context.write(orderBean, NullWritable.get());
        }
    }
}

（4）编写TableDriver类

/**
 * @author LS
 * @date 2021/12/15 11:18
 * @description
 */
public class TableDriver {
    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
        Configuration conf = new Configuration();
        Job job = Job.getInstance(conf);

        job.setJarByClass(TableDriver.class);
        job.setMapperClass(TableMapper.class);
        job.setReducerClass(TableReducer.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(TableBean.class);
        job.setOutputKeyClass(TableBean.class);
        job.setOutputValueClass(NullWritable.class);

        FileInputFormat.setInputPaths(job,new Path("F:\\input"));
        FileOutputFormat.setOutputPath(job,new Path("F:\\table"));
        job.waitForCompletion(true);
    }
}

        缺点：这种方式中，合并的操作是在Reduce阶段完成，Reduce端的处理压力太大，Map节点的运算负载则很低，资源利用率不高，且在Reduce阶段极易产生数据倾斜。

二、Map Join

Map Join适用于一张表十分小、一张表很大的场景。

表 订单数据表t_order

id	pid	amount
1001	01	1
1002	02	2
1003	03	3
1004	01	4
1005	02	5
1006	03	6

表 商品信息表t_product

pid	pname
01	小米
02	华为
03	格力

将商品信息表中数据根据商品pid合并到订单数据表中。

表 最终数据形式

id	pname	amount
1001	小米	1
1004	小米	4
1002	华为	2
1005	华为	5
1003	格力	3
1006	格力	6

实例：

（1）先在MapJoinDriver驱动类中添加缓存文件

/**
 * @author LS
 * @date 2021/12/15 18:12
 * @description
 */
public class MapJoinDriver {
    public static void main(String[] args) throws IOException, URISyntaxException, ClassNotFoundException, InterruptedException {
        Configuration conf = new Configuration();
        Job job = Job.getInstance(conf);

        job.setJarByClass(MapJoinDriver.class);
        job.setMapperClass(MapJoinMapper.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(NullWritable.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(NullWritable.class);
        // 加载缓存数据
        job.addCacheFile(new URI("file:///F:/pd.txt"));
        //不进行reduce过程
        job.setNumReduceTasks(0);

        FileInputFormat.setInputPaths(job,new Path("F:\\input"));
        FileOutputFormat.setOutputPath(job,new Path("F:\\table"));

        job.waitForCompletion(true);
    }
}

（2）在MapJoinMapper类中的setup方法中读取缓存文件

/**
 * @author LS
 * @date 2021/12/15 18:14
 * @description
 */
public class MapJoinMapper extends Mapper<LongWritable,Text,Text,NullWritable> {
   private Map<String,String> pdMap = new HashMap<>();
   private Text outK = new Text();

    @Override
    protected void setup(Context context) throws IOException, InterruptedException {
        //任务开始前将pd数据缓存进pdMap
        //通过缓存文件得到小表数据pd.txt
        URI[] cacheFiles = context.getCacheFiles();
        Path path = new Path(cacheFiles[0]);
        //获取文件系统对象,并开流
        FileSystem fs = FileSystem.get(context.getConfiguration());
        FSDataInputStream fis = fs.open(path);
        //通过包装流转换为reader,方便按行读取
        BufferedReader Reader = new BufferedReader(new InputStreamReader(fis, "UTF-8"));
        String line;

        while ((line = Reader.readLine())!= null){
            String[] split = line.split("\t");
            pdMap.put(split[0],split[1]);
        }

        IOUtils.closeStreams(fis);
        IOUtils.closeStreams(fs);

    }

    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        String string = value.toString();
        String[] split = string.split("\t");
        //通过大表每行数据的pid,去pdMap里面取出pname
        String pname = pdMap.get(split[1]);
        //将大表每行数据的pid替换为pname
        outK.set(split[0]+"\t"+pname+"\t"+split[1]);
        context.write(outK,NullWritable.get());
    }
}

优点：

        在Map端缓存多张表，提前处理业务逻辑，这样增加Map端业务，减少Reduce端数据的压力，尽可能的减少数据倾斜。