Reduce Join工作原理
Reduce Join案例实操
2.需求分析
通过将关联条件作为Map输出的key,将两表满足Join条件的数据并携带数据所来源的文件信息,发往同一个ReduceTask,在Reduce中进行数据的串联
相关代码
TableBean.java
package MapReduceJoin;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparator;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
public class TableBean implements Writable,Comparable<TableBean> {
private String orderId = new String(); //订单编号
private String pid = new String(); //品牌编号
private int amount = 0; //数量
private String pname = new String(); //品牌名称
public String getFlag() {
return flag;
}
public void setFlag(String flag) {
this.flag = flag;
}
private String flag = new String(); //标志,数据来源
public String getOrderId() {
return orderId;
}
public void setOrderId(String orderId) {
this.orderId = orderId;
}
public String getPid() {
return pid;
}
public void setPid(String pid) {
this.pid = pid;
}
public int getAmount() {
return amount;
}
public void setAmount(int amount) {
this.amount = amount;
}
public String getPname() {
return pname;
}
public void setPname(String pname) {
this.pname = pname;
}
@Override
public void write(DataOutput out) throws IOException {
out.writeUTF(orderId);
out.writeUTF(pid);
out.writeInt(amount);
out.writeUTF(pname);
out.writeUTF(flag);
}
@Override
public void readFields(DataInput in) throws IOException {
this.orderId = in.readUTF();
this.pid = in.readUTF();
this.amount = in.readInt();
this.pname = in.readUTF();
this.flag = in.readUTF();
}
@Override
public String toString() {
return orderId + '\t' + pname + '\t' + amount;
}
@Override
public int compareTo(TableBean o) {
return this.pid.compareTo(o.getPid());
}
}
TableMapper.java
package MapReduceJoin;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import java.io.IOException;
public class TableMapper extends Mapper<LongWritable, Text,Text, TableBean> {
Text k = new Text(); //编号
TableBean tb = new TableBean();
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
FileSplit fs = (FileSplit) context.getInputSplit();
String fileName = fs.getPath().getName();
String[] fields = value.toString().split("\t");
if(fileName.equals("order.txt")){
tb.setOrderId(fields[0]);
tb.setPid(fields[1]);
tb.setAmount(Integer.parseInt(fields[2]));
tb.setPname("");
tb.setFlag("Order");
}else if(fileName.equals("pd.txt")){
tb.setOrderId("");
tb.setPid(fields[0]);
tb.setAmount(0);
tb.setPname(fields[1]);
tb.setFlag("PD");
}
k.set(tb.getPid());
context.write(k,tb);
}
}
TableReducer.java
package MapReduceJoin;
import org.apache.commons.beanutils.BeanUtils;
import org.apache.commons.lang.ObjectUtils;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
import org.codehaus.jackson.map.util.BeanUtil;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.List;
public class TableReducer extends Reducer<Text,TableBean,TableBean,NullWritable> {
@Override
protected void reduce(Text key, Iterable<TableBean> values, Context context) throws IOException, InterruptedException {
//1001 01 1 和 01 小米 结合输出为 1001 小米 数量
List<TableBean> list = new ArrayList<>();
String pname = "";
for(TableBean tb : values){
if (tb.getFlag().equals("Order")){
TableBean newTb = new TableBean();
try {
BeanUtils.copyProperties(newTb,tb);
list.add(newTb);
} catch (IllegalAccessException e) {
e.printStackTrace();
} catch (InvocationTargetException e) {
e.printStackTrace();
}
}else{
if(pname.isEmpty()){
pname = tb.getPname();
}
}
}
for(TableBean tb : list){
tb.setPname(pname);
context.write(tb,NullWritable.get());
}
}
}
结果
坑
在reduce函数中,如果直接list.add(tb);那么结果是不正确的。因为Iterable<TableBean> values
迭代器遍历的时候用tb赋值只是改变地址值,并不会真的把对象拷贝给你。所以使用BeanUtils.copyProperties(newTb,tb);