MR之MapJoin
前言
有一个商品文件和一个订单文件,其中订单文件记录了商品文件的编号,而商品文件中有商品的中文名称。
需求:需要输出一个订单文件,字段为:商品名称、商品总量、总金额。
一、实现思路
我们需要读取商品文件形成一个Map集合,再通过订单文件中的商品id,在Map集合中找到商品名称,再实现对应的wc统计就行了
二、具体代码
1.MapJoinOrderBean类
package com.hadoop.mapreduce.mapJoin;
import org.apache.hadoop.io.WritableComparable;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
/**
* @author LengQing
* @date 2020/5/3 - 15:20
* 实现将订单表和产品表数据合并
*/
public class MapJoinOrderBean implements WritableComparable<MapJoinOrderBean> {
private String o_id; // order表:订单id
private String p_id; // order表、product表:商品id
private Integer o_price; // order表:商品单价
private Integer o_num; // order表:商品数量
private String p_name; // product表:商品名称
public MapJoinOrderBean() {
super();
}
public MapJoinOrderBean(String o_id, String p_id, Integer o_price, Integer o_num, String p_name) {
this.o_id = o_id;
this.p_id = p_id;
this.o_price = o_price;
this.o_num = o_num;
this.p_name = p_name;
}
@Override
public String toString() {
return o_id + '\t' + p_name + '\t' + o_num + '\t' + o_price;
}
public String getO_id() {
return o_id;
}
public void setO_id(String o_id) {
this.o_id = o_id;
}
public String getP_id() {
return p_id;
}
public void setP_id(String p_id) {
this.p_id = p_id;
}
public Integer getO_price() {
return o_price;
}
public void setO_price(Integer o_price) {
this.o_price = o_price;
}
public Integer getO_num() {
return o_num;
}
public void setO_num(Integer o_num) {
this.o_num = o_num;
}
public String getP_name() {
return p_name;
}
public void setP_name(String p_name) {
this.p_name = p_name;
}
@Override
public void write(DataOutput out) throws IOException {
out.writeUTF(o_id);
out.writeUTF(p_id);
out.writeInt(o_price);
out.writeInt(o_num);
out.writeUTF(p_name);
}
@Override
public void readFields(DataInput in) throws IOException {
o_id = in.readUTF();
p_id = in.readUTF();
o_price = in.readInt();
o_num = in .readInt();
p_name = in.readUTF();
}
@Override
public int compareTo(MapJoinOrderBean o) {
return 0;
}
}
2.MapJoinMapper类
package com.hadoop.mapreduce.mapJoin;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URI;
import java.util.HashMap;
import java.util.Map;
/**
* @author LengQing
* @date 2020/5/3 - 15:07
*/
public class MapJoinMapper extends Mapper<LongWritable, Text, Text, MapJoinOrderBean> {
private Text outputKey = new Text();
private MapJoinOrderBean orderBean = new MapJoinOrderBean();
private Map<String, String> pdMap = new HashMap<>();
@Override
protected void setup(Context context) throws IOException, InterruptedException {
String cacheFile = "hdfs://mycluster:9000/datas/products.txt";
FileSystem fileSystem = FileSystem.get(URI.create(cacheFile), context.getConfiguration());
FSDataInputStream fsDataInputStream = fileSystem.open(new Path(cacheFile));
BufferedReader reader = new BufferedReader(new InputStreamReader(fsDataInputStream));
// 1 获取输入文件切片
//URI[] cacheFiles = context.getCacheFiles();
//String path = cacheFiles[0].getPath().toString();
//FileInputStream fileInputStream = new FileInputStream(path);
//InputStreamReader inputStreamReader = new InputStreamReader(fileInputStream, "UTF-8");
//BufferedReader reader = new BufferedReader(inputStreamReader);
String line;
while(StringUtils.isNotEmpty(line = reader.readLine())) {
// 2 切割
String[] fields = line.split("\t");
// 3 缓存数据到集合
pdMap.put(fields[0], fields[1]);
}
// 4 关流
reader.close();
}
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
String[] line = value.toString().split("\t");
orderBean.setO_id(line[0]);
orderBean.setP_id(line[1]);
orderBean.setO_num(Integer.parseInt(line[2]));
orderBean.setO_price(Integer.parseInt(line[3]));
orderBean.setP_name(pdMap.get(line[1]));
outputKey.set(pdMap.get(line[1]));
context.write(this.outputKey, orderBean);
}
}
3.MapJoinReduce类
package com.hadoop.mapreduce.mapJoin;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
import java.io.IOException;
/**
* @author LengQing
* @date 2020/5/3 - 15:08
*/
public class MapJoinReduce extends Reducer<Text, MapJoinOrderBean, Text, Text> {
private Text outputValue = new Text();
@Override
protected void reduce(Text key, Iterable<MapJoinOrderBean> values, Context context) throws IOException, InterruptedException {
int num = 0;
Double price = 0.0;
for (MapJoinOrderBean value : values) {
num += value.getO_num();
price += value.getO_price();
}
outputValue.set(num + "\t" + price);
context.write(key, this.outputValue);
}
}
4.MapJoinDriver类
package com.hadoop.mapreduce.mapJoin;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
/**
* @author LengQing
* @date 2020/5/3 - 14:29
* 需要输出一个订单文件,字段为:商品名称、商品总量、总金额。
*/
public class MapJoinDriver extends Configured implements Tool {
private Configuration conf = new Configuration();
@Override
public int run(String[] args) throws Exception {
// 实例化Job
Job job = Job.getInstance(conf, "reduceJoin");
job.setJarByClass(MapJoinDriver.class);
//job.addCacheFile(new URI("hdfs://mycluster:8020/datas/order/product.txt"));
// 1 input阶段
Path inputPath = new Path(args[0]);
FileInputFormat.setInputPaths(job, inputPath);
// 2 map阶段
job.setMapperClass(MapJoinMapper.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(MapJoinOrderBean.class);
// 3 shuffle阶段
//job.setGroupingComparatorClass(mapJoinGroup.class);
// 4 reduce阶段
job.setReducerClass(MapJoinReduce.class);
job.setOutputKeyClass(NullWritable.class);
job.setOutputValueClass(Text.class);
//job.setNumReduceTasks(0);
// 5 output阶段
Path outputPath = new Path(args[1]);
FileOutputFormat.setOutputPath(job, outputPath);
return job.waitForCompletion(true) ? 0 : 1;
}
public static void main(String[] args) {
try {
int status = ToolRunner.run(new MapJoinDriver(), args);
System.exit(status);
} catch (Exception e) {
e.printStackTrace();
}
}
}
三、运行注意点
注意cacheFile文件位置,文件不能过大
集群运行输入参数:1、输入文件名称 2、输出文件名称
四、个人运行结果
1、输入文件order.txt
2、product.txt文件
3、export结果
总结
3年前被问到的面试题,实现很简单,但是却很重要。