MapReduce的Driver工具类

通常开发MapReduce都要开发自定义的Driver类，那么下面提供了基础的Driver工具类

import com.sinobest.yn.incrupdate.prop.PropertiesUtil;
import com.sinobest.yn.incrupdate.security.SecurityUtil;
import org.apache.directory.shared.kerberos.exceptions.KerberosException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;

import static com.sinobest.yn.incrupdate.DrivableDriver.*;

/**
 * @author liulv
 * @date 2018/11/9
 */
public class MRTools {
    public static Configuration conf;

    public MRTools(Configuration conf){
        if(conf != null)
        this.conf = conf;
        else this.conf = new Configuration();
    }

    static {
        conf.addResource(new Path(PropertiesUtil.CONFIG_PATH + CORE_SITE));
        conf.addResource(new Path(PropertiesUtil.CONFIG_PATH + HDFS_SITE));
        conf.addResource(new Path(PropertiesUtil.CONFIG_PATH + YARN_SITE));
        conf.addResource(new Path(PropertiesUtil.CONFIG_PATH + MAPRED_SITE));
        try {
            //kerberos安全认证
            SecurityUtil.security(conf);
        } catch (IOException | KerberosException e) {
            e.printStackTrace();
        }
        System.setProperty("HADOOP_USER_NAME", "root");
    }

    public static void setConf(String key, Object value){
        if(value instanceof String){
            conf.set(key, (String) value);
        }else if(value instanceof Integer){
            conf.setInt(key, (Integer) value);
        }else if(value instanceof Long){
            conf.setLong(key, (Long) value);
        }else if(value instanceof Double){
            conf.setDouble(key, (Double) value);
        }else if(value instanceof Float){
            conf.setDouble(key, (Float) value);
        }
    }

    public static Job getJob() {
        Job job = null;
        try {
            job = Job.getInstance(conf);
        } catch (IOException e) {
            e.printStackTrace();
        }
        return job;
    }

    public static void setJarByClass(Class jarClass){
        getJob().setJarByClass(jarClass);
    }

    public static void setMapper(Job job, Class mapperClass, Class keyClass, Class valueClass) {
        job.setMapperClass(mapperClass);
        job.setMapOutputKeyClass(keyClass);
        job.setMapOutputValueClass(valueClass);
    }

    public static void setReduce(Job job, Class reduceClass, Class keyClass, Class valueClass) {
        job.setReducerClass(reduceClass);
        job.setMapOutputKeyClass(keyClass);
        job.setMapOutputValueClass(valueClass);
    }

    public static void setInput(Job job, Class<FileInputFormat> inputFormat, String path) {
        try {
           inputFormat.newInstance().addInputPaths(job, path);
        } catch (IOException | InstantiationException | IllegalAccessException e) {
            e.printStackTrace();
        }
    }

    public static void setOutPut(Job job, Class<FileOutputFormat> outputFormatClass, String path) {
        try {
            outputFormatClass.newInstance().setOutputPath(job, new Path(path));
        } catch (InstantiationException | IllegalAccessException e) {
            e.printStackTrace();
        }
    }
}