控制MapReduce输出文件个数及格式

控制MapReduce多文件输出

默认情况下MapReduce任务结束后一个reduce产生一个输出文件，文件名类似part-xxxxx, 有时为了方便后续对这些文件的处理，比如根据文件名import到不通的hive分区，我们需要控制reduce输出产生的文件名，让相同的reduce key写入同一个文件，此时可继承MultipleOutputFormat重载generateFileNameForKeyValue定制OutputFormat。

public static class MyMultipleOutputFormat extends MultipleOutputFormat<K, V>{

   protected String generateFileNameForKeyValue(K key, V value) {
        return String.format("%s_%s", key.getDt(), key.getHour());
   }

   abstract protected RecordWriter<K, V> getBaseRecordWriter(FileSystem fs,
      JobConf job, String path, Progressable arg3) throws IOException；
}

这样同一天相同小时的日志就被聚集到同一个文件，很容易导入到对应的小时分区。

控制MapReduce输出格式

reduce输出文件格式不同，存储和读取效率都会有差别，可以重载getBaseRecordWriter返回不同格式的Writer.

SequenceFileWriter

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.SequenceFile.CompressionType;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.util.ReflectionUtils;

public class ValSequenceFileWriter<K, V> extends RecordWriter<K, V> {

    private SequenceFile.Writer out = null;
    private FSDataOutputStream fsout;

    public ValSequenceFileWriter(TaskAttemptContext context, Class<?> codecClass, FSDataOutputStream fsout) throws IOException {
        Configuration conf = context.getConfiguration();

        CompressionCodec codec = (CompressionCodec) 
                ReflectionUtils.newInstance(codecClass, conf);
        this.fsout = fsout;
        out = SequenceFile.createWriter(
            conf, fsout,
            NullWritable.class,
            context.getOutputValueClass(),
            CompressionType.BLOCK,
            codec);
    }

    @Override
    public synchronized void close(TaskAttemptContext context) throws IOException,
            InterruptedException {
        out.close();
        fsout.close();
    }

    @Override
    public synchronized void write(K key, V value) throws IOException, InterruptedException {
        out.append(NullWritable.get(), value);
    }   
}

RCFileWriter

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.ql.io.RCFile;
import org.apache.hadoop.hive.serde2.columnar.BytesRefArrayWritable;
import org.apache.hadoop.hive.serde2.columnar.BytesRefWritable;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.util.ReflectionUtils;

import com.pplive.bip.metadata.hive.HiveTableInfo;

public class ValRCFileWriter<K, V> extends RecordWriter<K, V> {
    protected RCFile.Writer out;

    /*
     * buffered output array for output
     */
    private final BytesRefArrayWritable array;
    private final int numColumns;

    /**
     * construct
     * @param conf
     * @param codecClass
     * @param path
     * @throws IOException
     */
    public ValRCFileWriter(Configuration conf ,
            Class<?> codecClass, Path path) throws IOException {

        FileSystem fs = path.getFileSystem(conf);
        CompressionCodec codec = (CompressionCodec) 
            ReflectionUtils.newInstance(codecClass, conf);

        this.out = new RCFile.Writer(fs, conf, path, null, codec);
        numColumns = conf.getInt(RCFile.COLUMN_NUMBER_CONF_STR, 0);
        this.array = new BytesRefArrayWritable(numColumns);
    }

    @Override
    public synchronized void write(K key, V value) throws IOException {

        String[] fields = value.toString().split(HiveTableInfo.FIELD_DELIMITED, -1);
        for (int i = 0; i < fields.length && i < this.numColumns; i++) {
            array.set(i, new BytesRefWritable(fields[i].getBytes("UTF-8")));
        }
        out.append(array);
    }

    @Override
    public synchronized void close(TaskAttemptContext context) throws IOException,
            InterruptedException {
        out.close();        
    }
}

ORCFileWriter

import java.io.IOException;
import java.lang.reflect.Constructor;
import java.util.*;

import com.google.common.base.Throwables;
import com.pplive.bip.metadata.hive.HiveColumnInfo;
import com.pplive.bip.metadata.hive.HiveColumnType;
import com.pplive.bip.metadata.log.LogInfo;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.ql.io.orc.*;
import org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter;
import org.apache.hadoop.hive.serde2.objectinspector.*;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.*;
import org.apache.hadoop.hive.serde2.typeinfo.*;
import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.mapreduce.TaskAttemptContext;

import com.pplive.bip.metadata.hive.HiveTableInfo;

public class ValORCFileWriter<K, V> extends org.apache.hadoop.mapreduce.RecordWriter<K, V> {

    private final static String MapTypeString = "MAP<STRING,STRING>";

    private final static String ArrayTypeString = "ARRAY<STRING>";

    private  OrcSerde orcSerde;

    private RecordWriter writer;

    private final SettableStructObjectInspector tableInspector;

    private final List<StructField> structFields;

    private final Object orcRow;

    private final int numColumns;

    private static Constructor<? extends RecordWriter> getOrcWriterConstructor()
    {
        try {
            String writerClassName = OrcOutputFormat.class.getName() + "$OrcRecordWriter";
            Constructor<? extends RecordWriter> constructor = OrcOutputFormat.class.getClassLoader()
                    .loadClass(writerClassName).asSubclass(RecordWriter.class)
                    .getDeclaredConstructor(Path.class, OrcFile.WriterOptions.class);
            constructor.setAccessible(true);
            return constructor;
        }
        catch (ReflectiveOperationException e) {
            throw Throwables.propagate(e);
        }
    }



    private static String  getHiveType(HiveColumnInfo column) {
        String hiveType;
        HiveColumnType columnType = column.getHiveType();
        if (columnType == HiveColumnType.Map)
            hiveType =  MapTypeString;
        else if (columnType == HiveColumnType.Complex)
            hiveType =  column.getColumnTypeValue();
        else if (columnType == HiveColumnType.Array)
            hiveType =  ArrayTypeString;
        else
            hiveType = column.getHiveType().toString();

        return hiveType.toLowerCase();
    }

    private static ObjectInspector getObjectInspector(String hiveType) {
        TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(hiveType.toLowerCase());
        return TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(typeInfo);
    }

    /**
     * construct
     * @param conf
     * @param path
     * @throws IOException
     */
    public ValORCFileWriter(Configuration conf, LogInfo logInfo, Path path) throws IOException {
        HiveTableInfo hiveTableInfo = logInfo.getHiveTable();
        List<String> columnNames = new ArrayList<String>();
        List<String> hiveTypeNames = new ArrayList<String>();
        List<ObjectInspector> columnInspectors = new ArrayList<ObjectInspector>();

        for (HiveColumnInfo columnInfo: hiveTableInfo.getHiveColumns()) {
            columnNames.add(columnInfo.getName());
            String hiveType = getHiveType(columnInfo);
            hiveTypeNames.add(hiveType);
            columnInspectors.add(getObjectInspector(hiveType));
        }

        this.tableInspector = ObjectInspectorFactory.getStandardStructObjectInspector(columnNames, columnInspectors);
        this.structFields = (List<StructField>) tableInspector.getAllStructFieldRefs();

        Constructor<? extends RecordWriter> writerConstructor = getOrcWriterConstructor();
        try {
            this.writer = writerConstructor.newInstance(path, OrcFile.writerOptions(conf).inspector(this.tableInspector));
        } catch (ReflectiveOperationException  e) {
            throw new RuntimeException("Failed to create writer", e);
        }
        this.orcSerde = new OrcSerde();
        Properties properties = new Properties();
        properties.setProperty(hive_metastoreConstants.META_TABLE_COLUMNS, StringUtils.join(columnNames, ','));
        properties.setProperty(hive_metastoreConstants.META_TABLE_COLUMN_TYPES, StringUtils.join(hiveTypeNames, ':'));
        this.orcSerde.initialize(conf, properties);
        this.orcRow = tableInspector.create();
        this.numColumns = hiveTableInfo.getHiveColumnCount();
    }

    @Override
    public synchronized void write(K key, V value) throws IOException {
        String[] fields = value.toString().split(HiveTableInfo.FIELD_DELIMITED, -1);
        for (int i = 0; i < fields.length && i < this.numColumns; i++) {
            StructField sf = structFields.get(i);
            tableInspector.setStructFieldData(orcRow, sf, getJavaObject(fields[i], sf.getFieldObjectInspector()));
        }
        this.writer.write(this.orcSerde.serialize(orcRow, tableInspector));
    }

    private Object getJavaObject(String value, ObjectInspector oi) {
        Class clazz = oi.getClass();
        if(value.isEmpty()) {
            return null;
        }
        Object o;
        try {
            if (clazz == JavaShortObjectInspector.class) {
                o = new Short(value);
            } else if (clazz == JavaIntObjectInspector.class) {
                o = new Integer(value);
            }  else if (clazz == JavaLongObjectInspector.class) {
                o = new Long(value);
            } else if (clazz == JavaStringObjectInspector.class) {
                o = value;
            } else if (clazz == JavaFloatObjectInspector.class) {
                o = new Float(value);
            } else if (clazz == JavaDoubleObjectInspector.class) {
                o = new Double(value);
            } else if (clazz == StandardListObjectInspector.class) {
                ObjectInspector elementObjectInspector = ((StandardListObjectInspector)oi).getListElementObjectInspector();
                String[] vs = value.split(",");
                List l  = new ArrayList();
                for(String v: vs) {
                    l.add(getJavaObject(v,elementObjectInspector));
                }
                o = l;
            } else if (clazz == StandardMapObjectInspector.class) {
                ObjectInspector keyObjectInspector = ((StandardMapObjectInspector)oi).getMapKeyObjectInspector();
                ObjectInspector valueObjectInspector = ((StandardMapObjectInspector)oi).getMapValueObjectInspector();
                Map m = new HashMap();
                if(!value.isEmpty()) {
                    String[] vs = value.split(",");
                    for(String v: vs) {
                        String[] kv = v.split(":");
                        if(kv.length == 2) {
                            m.put(getJavaObject(kv[0],keyObjectInspector), getJavaObject(kv[1],valueObjectInspector));
                        }
                    }
                }
                o = m;
            } else if (clazz == StandardStructObjectInspector.class) {
                StandardStructObjectInspector soi = (StandardStructObjectInspector)oi;
                List<StructField> fields = (List<StructField>) soi.getAllStructFieldRefs();
                ArrayList result = (ArrayList)soi.create();
                String[] vs = value.split(":");
                if(vs.length == fields.size()) {
                    for (int i = 0; i < fields.size(); i++) {
                        StructField structField = fields.get(i);
                        soi.setStructFieldData(result,structField,getJavaObject(vs[i], structField.getFieldObjectInspector()));
                    }
                }
                o = result;
            } else {
                throw new RuntimeException("invalid object ObjectInspector" + oi.toString());
            }
        }catch (NumberFormatException e) {
            o = null;
        }

        return o;
    }

    @Override
    public synchronized void close(TaskAttemptContext context) throws IOException, InterruptedException {
        writer.close(false);
    }

三种不同的Writer输出的数据查询效率依次变高，ORC最常用，效率也最高。