MapReduce - Map Join 案例

数据基于https://blog.csdn.net/u012321968/article/details/107863294

使用场景

Map Join适用于一张表十分小、一张表很大的场景。

优点

思考：在Reduce端处理过多的表，非常容易产生数据倾斜。怎么办？
在Map端缓存多张表，提前处理业务逻辑，这样增加Map端业务，减少Reduce端数据的压力，尽可能的减少数据倾斜。

具体办法：采用DistributedCache

（1）在Mapper的setup阶段，将文件读取到缓存集合中。
（2）在驱动函数中加载缓存。

// 缓存普通文件到Task运行节点。
job.addCacheFile(new URI("file:///E:/cache/........"));

思路

相关代码

TableMapper.java

package MapReduceMapJoin;

import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.*;
import java.net.URI;
import java.util.HashMap;
import java.util.Map;

public class TableMapper extends Mapper<LongWritable, Text,TableBean, NullWritable> {
    private Map<String, String> pdMap = new HashMap<>();
    TableBean tb = new TableBean();

    @Override
    protected void setup(Context context) throws IOException, InterruptedException {
        URI cacheFile = context.getCacheFiles()[0];
        String path = cacheFile.getPath();
        BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(path),"UTF-8"));
        String line;
        while(StringUtils.isNotEmpty(line = br.readLine())){
            String[] split = line.split("\t");
            pdMap.put(split[0],split[1]);
        }

        br.close();
    }

    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        String[] split = value.toString().split("\t");
        tb.setOrderId(split[0]);
        tb.setPname(pdMap.get(split[1]));
        tb.setAmount(Integer.parseInt(split[2]));

        context.write(tb,NullWritable.get());
    }
}

TableDriver.java

package MapReduceMapJoin;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

public class TableDriver {
  public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException, URISyntaxException {
    Configuration conf = new Configuration();
    Job job = Job.getInstance(conf);

    job.setJarByClass(TableDriver.class);
    job.setMapperClass(TableMapper.class);

    //不需要reduce
    job.setNumReduceTasks(0);

    //加载缓存数据
    job.setCacheFiles(new URI[] {new URI("file:///F:/Codes/JavaCodes/MapReduceLearning/testdata/Join/pd.txt")});

    job.setMapOutputKeyClass(TableBean.class);
    job.setMapOutputValueClass(NullWritable.class);

    FileInputFormat.setInputPaths(job,new Path("F:/Codes/JavaCodes/MapReduceLearning/testdata/Join/order.txt"));
    FileOutputFormat.setOutputPath(job,new Path("F:/Codes/JavaCodes/MapReduceLearning/testdata/Join/output"));

    System.exit(job.waitForCompletion(true) ? 0 : 1);
  }
}