文章标题

Hive 自定 InputFormat 解析 Hadoop MR 产生的SequenceFile 文件
注意： Hive 使用的MRV1
1.定义InputFormat

package cn.gitv.bi.log.analysis.io;

import java.io.IOException;

import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.MapFile;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.FileSplit;
import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.Reporter;

public class SequenceFileKeyInputFormat<K, V> extends FileInputFormat<K, V> {

    public SequenceFileKeyInputFormat() {
        setMinSplitSize(SequenceFile.SYNC_INTERVAL);
    }

    @SuppressWarnings("deprecation")
    @Override
    protected FileStatus[] listStatus(JobConf job) throws IOException {
        FileStatus[] files = super.listStatus(job);
        for (int i = 0; i < files.length; i++) {
            FileStatus file = files[i];
            if (file.isDir()) { // it's a MapFile
                Path dataFile = new Path(file.getPath(), MapFile.DATA_FILE_NAME);
                FileSystem fs = file.getPath().getFileSystem(job);
                // use the data file
                files[i] = fs.getFileStatus(dataFile);
            }
        }
        return files;
    }

    @SuppressWarnings("unchecked")
    public RecordReader<K, V> getRecordReader(InputSplit split, JobConf job,
            Reporter reporter) throws IOException {

        reporter.setStatus(split.toString());

        return (RecordReader<K, V>) new KVTextReader(job, (FileSplit) split);
    }
}

2.定义RecordReader

package cn.gitv.bi.log.analysis.io;

import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapred.FileSplit;
import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.util.ReflectionUtils;

public abstract class SequenceFileKeyRecordReader<K, V> implements
        RecordReader<K, BytesWritable> {

    private SequenceFile.Reader in;
    private long start;
    private long end;
    private boolean more = true;
    protected Configuration conf;

    @SuppressWarnings("deprecation")
    public SequenceFileKeyRecordReader(Configuration conf, FileSplit split)
            throws IOException {
        Path path = split.getPath();
        FileSystem fs = path.getFileSystem(conf);
        this.in = new SequenceFile.Reader(fs, path, conf);
        this.end = split.getStart() + split.getLength();
        this.conf = conf;

        if (split.getStart() > in.getPosition())
            in.sync(split.getStart()); // sync to start

        this.start = in.getPosition();
        more = start < end;
    }

    @SuppressWarnings("rawtypes")
    public Class getKeyClass() {
        return in.getKeyClass();
    }

    @SuppressWarnings("rawtypes")
    public Class getValueClass() {
        return in.getValueClass();
    }

    @SuppressWarnings("unchecked")
    public K createKey() {
        return (K) ReflectionUtils.newInstance(getKeyClass(), conf);
    }

    public BytesWritable createValue() {
        return new BytesWritable();
    }

    public float getProgress() throws IOException {
        if (end == start) {
            return 0.0f;
        } else {
            return Math.min(1.0f, (in.getPosition() - start)
                    / (float) (end - start));
        }
    }

    public synchronized long getPos() throws IOException {
        return in.getPosition();
    }

    protected synchronized void seek(long pos) throws IOException {
        in.seek(pos);
    }

    public synchronized void close() throws IOException {
        in.close();
    }

    public boolean next(K key, BytesWritable value) throws IOException {
        if (!more)
            return false;

        long pos = in.getPosition();
        @SuppressWarnings("unchecked")
        V trueValue = (V) ReflectionUtils.newInstance(in.getValueClass(), conf);
        boolean remaining = in.next((Writable) key, (Writable) trueValue);
        if (remaining)
            combineKeyValue(key, trueValue, value);
        if (pos >= end && in.syncSeen()) {
            more = false;
        } else {
            more = remaining;
        }
        return more;
    }

    protected abstract void combineKeyValue(K key, V trueValue,
            BytesWritable newValue);

    protected void combineKeyValue(Text key, LongWritable trueValue,
            BytesWritable newValue) {
        // TODO I think we need to use straight bytes--I'm not sure this works?
        /*
         * StringBuilder builder = new StringBuilder(); builder.append(key);
         * builder.append('\001'); builder.append(trueValue.get());
         * newValue.set(new BytesWritable(builder.toString().getBytes()) );
         */
    }
}

3.定义RecordReader的读取规则，此类继承上述定义的RecorderReader

package cn.gitv.bi.log.analysis.io;

import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.FileSplit;

public class KVTextReader extends SequenceFileKeyRecordReader<Text, Text> {

    public KVTextReader(Configuration conf, FileSplit split) throws IOException {
        super(conf, split);

    }

    @Override
    protected void combineKeyValue(Text key, Text trueValue,
            BytesWritable newValue) {
        StringBuilder builder = new StringBuilder();
        builder.append(key);
        builder.append('\001');
        builder.append(trueValue);
        newValue.set(new BytesWritable(builder.toString().getBytes()));

    }

    @Override
    public BytesWritable createValue() {
        return new BytesWritable();
    }

}

注：上述代码，只能解析HadoopSequenceFile中Key 为Text类型，Value 为Text 类型的 SequenceFile文件。如果要解析其他类型，请自行修改上述第三个代码类，根据情况修改为自己需要的KV类型。