Hadoop Left Join2

我在上一篇博客Hadoop实现LeftJoin操作上已经分享过一种实现LeftJoin操作的方法。这次分享一种自定义数据类型来实现LeftJoin，该方法相对与之前的方法要更高效。简单来说，之前分享的方法是把两张表先按照同一种格式去map，也就是说无论是员工表还是公司表都是一样处理，只不过在Reducer时进行判断，将原本为null的内容替换，所以会有不少的浪费。而本次介绍的方法，自定义一种数据类型，对同一个公司，把公司表放在员工表前面，这样每个Key的第一个一定是公司，不需要做判断，也不需要重复存储一个公司。

说了这么多废话，接下来正式介绍本方法。首先将测试数据放在这方便理解：
employee
jd,david
jd,mike
tb,mike
tb,lucifer
elong,xiaoming
elong,ali
tengxun,xiaoming
tengxun,lilei
xxx,aaa

salary
jd,1600
tb,1800
elong,2000
tengxun,2200

首先定义一个自定义的键值，该键值作为Map过程的输出键值，包含一个Cid，表示公司编号(在这里就是公司名称，后面不再赘述），一个isEmployee,代表是否为employee。hadoop规定，若是自定义键值类型，必须:
1、实现Writable接口
实现readFields,将输入流字节反序列化

public void readFields(DataInput in) throws IOException {
        this.cid = in.readUTF() ;
        this.isEmployee = in.readBoolean();

    }

实现write,把每个对象序列化到输出

public void write(DataOutput out) throws IOException {
        out.writeUTF(cid);
        out.writeBoolean(isEmployee);

    }

注意readFields和write里面的顺序必须相同

２、重写compareTo函数
重写compareTo函数，自定义比较函数

public int compareTo(EmployeeKey o) {
        if(this.cid .equals(o.cid)){
            if(this.getIs()==o.getIs()){
                return 0 ;
            }
            else if(this.getIs()==true){
                return 1 ;
            }
            else return -1 ;
        }
        else {
            return (this.getCid().compareTo(o.getCid())>0)?1:-1 ;
        }
    }

3、重写group
如果不重写group，由于公司表和职员表里面的isEmployee不同，同一个公司的公司表和职员表将不在一个List中，这当然不是我们想要的结果。

因此要实现一个继承WritableComparator的类，并在主类中设置setGroupingComparatorClass

class GroupComParator extends WritableComparator{
    public GroupComParator(){
        super(EmployeeKey.class,true) ;
    }

    @SuppressWarnings("rawtypes")
    @Override
    public int compare(WritableComparable a , WritableComparable b) {
        EmployeeKey aKey = (EmployeeKey) a ;
        EmployeeKey bKey = (EmployeeKey) b ;
        if(aKey.getCid().equalsIgnoreCase(bKey.getCid())){
            return 0 ;
        }
        else{
            return ( aKey.getCid().compareTo(bKey.getCid()) == -1)?1:-1 ;
        }
    }   
} 

自定义好mapper的输出键类型后，就执行mapper操作

class MyMapper extends Mapper<Object,Text,EmployeeKey,Employee> {
    protected void map(Object key,Text value ,Context context)throws IOException,InterruptedException{
        String filePath = ( (FileSplit)context.getInputSplit()).getPath().toString()  ;
        String val = value.toString() ;
        if(val == null || val == "") return ;
        EmployeeKey outKey = new EmployeeKey() ;
        if(filePath.indexOf("employee")!=-1){
            String[] line = val.split(",") ;
            if(line.length<2)return ;
            Employee em = new Employee() ;
            outKey.setCid(line[0]);
            outKey.setIs(true);
            em.setCid(line[0]);
            em.setEid(line[1]);
            em.setSalary("-1");
            context.write(outKey,em);
        }
        else if(filePath.indexOf("salary")!=-1){
            String[] line = val.split(",") ;
            if(line.length<2) return ;
            Employee em = new Employee() ;
            outKey.setCid(line[0]); 
            outKey.setIs(false);
            em.setCid(line[0]) ;
            em.setEid("-1");
            em.setSalary(line[1]) ;
            context.write(outKey,em);
        }
    }
}

以上过程会将同一个cid的放在一个list里面，接下来只要在reducer过程中将第一个元素的salary填入其他的元素中即可合并两张表。

package join;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableComparator;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;


class EmployeeKey implements WritableComparable<EmployeeKey>{

    private String cid ;
    private boolean isEmployee ;

    public EmployeeKey()
    {

    }
    public EmployeeKey(String cid ,boolean isEmployee){
        this.cid = cid ;
        this.isEmployee = isEmployee;
    }

    public void readFields(DataInput in) throws IOException {
        this.cid = in.readUTF() ;
        this.isEmployee = in.readBoolean();

    }


    public void write(DataOutput out) throws IOException {
        out.writeUTF(cid);
        out.writeBoolean(isEmployee);

    }


    public int compareTo(EmployeeKey o) {
        if(this.cid .equals(o.cid)){
            if(this.getIs()==o.getIs()){
                return 0 ;
            }
            else if(this.getIs()==true){
                return 1 ;
            }
            else return -1 ;
        }
        else {
            return (this.getCid().compareTo(o.getCid())>0)?1:-1 ;
        }
    }

    public String getCid(){
        return this.cid ;
    }
    public boolean getIs(){
        return this.isEmployee ;
    }
    public void setCid(String cid){
        this.cid = cid;
    }
    public void setIs(boolean isEmployee){
        this.isEmployee  = isEmployee;
    }

}

class Employee implements Writable{

    private String cid ;
    private String eid ;
    private String salary ;

    public void readFields(DataInput in) throws IOException {
        this.cid = in.readUTF() ;
        this.eid = in.readUTF() ;
        this.salary = in.readUTF() ;
    }

    public void write(DataOutput out) throws IOException {
        out.writeUTF(cid) ;
        out.writeUTF(eid) ;
        out.writeUTF(salary);

    }

    public String getCid() {
        return cid;
    }

    public void setCid(String cid) {
        this.cid = cid;
    }

    public String getEid() {
        return eid;
    }

    public void setEid(String eid) {
        this.eid = eid;
    }

    public String getSalary() {
        return salary;
    }

    public void setSalary(String salary) {
        this.salary = salary;
    }


}


class GroupComParator extends WritableComparator{
    public GroupComParator(){
        super(EmployeeKey.class,true) ;
    }

    @SuppressWarnings("rawtypes")
    @Override
    public int compare(WritableComparable a , WritableComparable b) {
        EmployeeKey aKey = (EmployeeKey) a ;
        EmployeeKey bKey = (EmployeeKey) b ;
        if(aKey.getCid().equalsIgnoreCase(bKey.getCid())){
            return 0 ;
        }
        else{
            return ( aKey.getCid().compareTo(bKey.getCid()) == -1)?1:-1 ;
        }
    }   
} 

class MyMapper extends Mapper<Object,Text,EmployeeKey,Employee> {
    protected void map(Object key,Text value ,Context context)throws IOException,InterruptedException{
        String filePath = ( (FileSplit)context.getInputSplit()).getPath().toString()  ;
        String val = value.toString() ;
        if(val == null || val == "") return ;
        EmployeeKey outKey = new EmployeeKey() ;
        if(filePath.indexOf("employee")!=-1){
            String[] line = val.split(",") ;
            if(line.length<2)return ;
            Employee em = new Employee() ;
            outKey.setCid(line[0]);
            outKey.setIs(true);
            //System.out.println(line[1]+" outkey");
            em.setCid(line[0]);
            em.setEid(line[1]);
            em.setSalary("-1");
            context.write(outKey,em);
        }
        else if(filePath.indexOf("salary")!=-1){
            String[] line = val.split(",") ;
            if(line.length<2) return ;
            Employee em = new Employee() ;
            outKey.setCid(line[0]); 
            outKey.setIs(false);
            em.setCid(line[0]) ;
            em.setEid("-1");
            em.setSalary(line[1]) ;
            context.write(outKey,em);
        }


    }
}

class MyReducer extends Reducer<EmployeeKey,Employee,Object,Text> {
    protected void reduce(EmployeeKey key,Iterable<Employee> vals ,Context context)throws IOException,InterruptedException{
        EmployeeKey eKey = key ;
        String salary="" ;
        for(Employee val : vals){           
            if(eKey.getIs() == true){
                val.setSalary(salary);
                if(salary!="")
                context.write( null,new Text(val.getCid()+" "+val.getEid()+" "+val.getSalary()));
                else
                context.write( null,new Text(val.getCid()+" "+val.getEid()+" "+"NULL"));
            }
            else{
                salary =  new String(val.getSalary());
            }
        }
    }
}
public class LeftJoin {

    public static void main(String[] args) throws Exception{
        Configuration config = new Configuration();
        Job job = new Job(config,"Left Join") ;
        FileInputFormat.setInputPaths(job, new Path(args[0])) ;
        FileOutputFormat.setOutputPath(job, new Path(args[1]));
        job.setGroupingComparatorClass(GroupComParator.class);
        job.setJarByClass(LeftJoin.class);
        job.setMapperClass(MyMapper.class);
        job.setMapOutputKeyClass(EmployeeKey.class);
        job.setMapOutputValueClass(Employee.class);

        job.setReducerClass(MyReducer.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(Text.class);

        System.exit(job.waitForCompletion(true)?0:1);


    }
}