通过MapReduce把Hive表数据导入到HBase

由于Hive查询速度比较慢，进行了表分区使用Impala也是很满意，所以为了公司业务展示，需要测试使用HBase的查询速度怎么样，头一件事就是把HIVE的数据导入到HBase中，搜了半天也没搜到到底该怎么搞，也有说能用Sqoop的，可是没找到资料，只好自己用MapReduce实现。

话不多说，逻辑很简单，只是用了Map，直接上代码。

public class Hive2HBase {

    /**
     * Mapper
     */
    static class ImportMapper extends Mapper<LongWritable, Text, ImmutableBytesWritable, Put> {

        @Override
        public void map(LongWritable offset, Text value, Context context) {
            String[] splited = value.toString().split("\t");
            if (splited.length != 4)
                return;
            try {

                byte[] rowkey = Bytes.toBytes(splited[0]);// id作为rowkey

                Put put = new Put(rowkey);
                // 为了省事直接列名为log1...log4

                for (int j = 0; j < splited.length; j++) {
                    put.addColumn(Bytes.toBytes(HConfiguration.colFamily), Bytes.toBytes("log" + j),
                            Bytes.toBytes(splited[j]));
                }

                context.write(new ImmutableBytesWritable(rowkey), put);

            } catch (NumberFormatException e) {
                System.out.println("出错了" + e.getMessage());
            } catch (IOException e) {
                e.printStackTrace();
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }
    }

    /**
     * Main
     * 
     * @param args
     * @throws Exception
     */
    public static void main(String[] args) throws Exception {

        Configuration configuration = new Configuration();
        // 设置zookeeper
        configuration.set("hbase.zookeeper.quorum", HConfiguration.hbase_zookeeper_quorum);
        configuration.set("hbase.zookeeper.property.clientPort", "2181");
        // 设置hbase表名称
        configuration.set(TableOutputFormat.OUTPUT_TABLE, HConfiguration.tableName);
        // 将该值改大，防止hbase超时退出
        configuration.set("dfs.client.socket-timeout", "180000");

        MRDriver myDriver = MRDriver.getInstance();

        try {
            //创建表
            myDriver.createTableIfExistDelete(HConfiguration.tableName, HConfiguration.colFamily);
        } catch (Exception e) {
            e.printStackTrace();
        }

        Job job = new Job(configuration, "HBaseBatchImport");

        job.setJarByClass(Hive2HBase.class);
        job.setMapperClass(ImportMapper.class);
        // 设置map的输出，不设置reduce的输出类型
        job.setMapOutputKeyClass(ImmutableBytesWritable.class);
        job.setMapOutputValueClass(Writeable.class);
        job.setNumReduceTasks(0);

        job.setInputFormatClass(TextInputFormat.class);
        // 不再设置输出路径，而是设置输出格式类型
        job.setOutputFormatClass(TableOutputFormat.class);
        // hive表路径
        FileInputFormat.setInputPaths(job, "hdfs://172.*.*.2:8022/user/hive/warehouse/sample_07");

        job.waitForCompletion(true);
    }
}