HBase加载数据，和hive整合

HBase加载数据原理：利用HBase的数据信息按照特定格式存储在hdfs内这一原理，直接在HDFS中生成持久化的HFile数据格式文件，然后上传至适当位置；

HFile文件：是数据的实际存储格式，他是二进制文件。StoreFile对HFile进行了封装。HBase的数据在底层文件中时以KeyValue键值对的形式存储的，HBase没有数据类型，HFile中存储的是字节，这些字节按字典序排列。

自定义Map类：
public class BulkLoadMapper extends Mapper<LongWritable,Text,ImmutableBytesWritable,Put> {
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
    String[] split = value.toString().split("\t");
    Put put = new Put(split[0].getBytes()); 	//rowkey
    put.addColumn("f1".getBytes(),"name".getBytes(),split[1].getBytes()); //列族\列名\值
    put.addColumn("f1".getBytes(),"age".getBytes(),split[2].getBytes());
    context.write(new ImmutableBytesWritable(split[0].getBytes()),put);
}
}
主类main：
public class BulkLoadMain extends Configured implements Tool {
@Override
public int run(String[] args) throws Exception {
    Configuration conf = super.getConf();
    Connection connection = ConnectionFactory.createConnection(conf);
    Table table = connection.getTable(TableName.valueOf("myuser2"));//目标的hbase表
    Job job = Job.getInstance(conf, "bulkLoad");
    //读取文件，解析成key,value对
    job.setInputFormatClass(TextInputFormat.class);
    TextInputFormat.addInputPath(job,new Path("hdfs://node01:8020/hbase/input"));
    //定义我们的mapper类
    job.setMapperClass(BulkLoadMapper.class);
    job.setMapOutputKeyClass(ImmutableBytesWritable.class);
    job.setMapOutputValueClass(Put.class);
    //reduce过程也省掉
    /**
     * Job job, Table table, RegionLocator regionLocator
     *  使用configureIncrementalLoad来进行配置我们的HFile加载到哪一个表里面的哪一个列族里面去
     */
//设置数据增量导入        HFileOutputFormat2.configureIncrementalLoad(job,table,connection.getRegionLocator(TableName.valueOf("myuser2")));
    //设置我们的输出类型，将我们的数据输出成为HFile格式
    job.setOutputFormatClass(HFileOutputFormat2.class);
    //设置我们的输出路径
    HFileOutputFormat2.setOutputPath(job,new Path("hdfs://node01:8020/hbase/hfile_out"));
    boolean b = job.waitForCompletion(true);
    return b?0:1;
}
public static void main(String[] args) throws Exception {
    Configuration configuration = HBaseConfiguration.create();
    configuration.set("hbase.zookeeper.quorum", "node01:2181,node02:2181");
    int run = ToolRunner.run(configuration, new BulkLoadMain(), args);
    System.exit(run);
}
}


加载到hbase:
public class LoadData {
public static void main(String[] args) throws Exception {
    Configuration configuration = HBaseConfiguration.create();
    configuration.set("hbase.zookeeper.property.clientPort", "2181");
    configuration.set("hbase.zookeeper.quorum", "node01,node02,node03");

    Connection connection =  ConnectionFactory.createConnection(configuration);
    Admin admin = connection.getAdmin();
    Table table = connection.getTable(TableName.valueOf("myuser2"));
    LoadIncrementalHFiles load = new LoadIncrementalHFiles(configuration);
    load.doBulkLoad(new Path("hdfs://node01:8020/hbase/output_hfile"), admin,table,connection.getRegionLocator(TableName.valueOf("myuser2")));
}
}

yarn jar hbase-mapreduce-2.0.0.jar 路径1 hbase目标表name

load data inpath ‘/hbase/hfile_out’ into table myuser2

Hbase和hive整合：

1、拷贝hbase的五个依赖jar包到hive的lib目录下

ln -s /export/servers/hbase-2.0.0/lib/hbase-client-2.0.0.jar /export/servers/apache-hive-2.1.0-bin/lib/hbase-client-2.0.0.jar
ln -s /export/servers/hbase-2.0.0/lib/hbase-hadoop2-compat-2.0.0.jar /export/servers/apache-hive-2.1.0-bin/lib/hbase-hadoop2-compat-2.0.0.jar
ln -s /export/servers/hbase-2.0.0/lib/hbase-hadoop-compat-2.0.0.jar /export/servers/apache-hive-2.1.0-bin/lib/hbase-hadoop-compat-2.0.0.jar
ln -s /export/servers/hbase-2.0.0/lib/hbase-it-2.0.0.jar /export/servers/apache-hive-2.1.0-bin/lib/hbase-it-2.0.0.jar
ln -s /export/servers/hbase-2.0.0/lib/hbase-server-2.0.0.jar /export/servers/apache-hive-2.1.0-bin/lib/hbase-server-2.0.0.jar 

2、修改hive配置文件hive-site.xml

<property>
<name>hive.zookeeper.quorum</name>
<value>node01,node02,node03</value>
</property>

<property>
<name>hbase.zookeeper.quorum</name>
<value>node01,node02,node03</value>
</property>

3、修改hive-env.sh配置文件添加配置

export HADOOP_HOME=/export/servers/hadoop-2.7.5
export HBASE_HOME=/export/servers/hbase-2.0.0
export HIVE_CONF_DIR=/export/servers/apache-hive-2.1.0-bin/conf

4、我们可以创建一个hive的管理表与hbase当中的表进行映射，hive管理表当中的数据，都会存储到hbase上面去

4.1创建hive数据库与hive对应的数据库表

create database course;
use course;
create external table if not exists course.score(id int,cname string,score int) row format 	delimited fields terminated by '\t' stored as textfile ;

4.2hive当中创建内部表（course数据库名）

create table course.hbase_score(id int,cname string,score int)  
stored by 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'  
with serdeproperties("hbase.columns.mapping" = "cf:name,cf:score") 
tblproperties("hbase.table.name" = "hbase_score");

4.3通过insert overwrite select 插入数据

insert overwrite table course.hbase_score select id,cname,score from course.score;

4.4进入hbase的客户端查看表hbase_score，并查看当中的数据

hbase(main):023:0> list
TABLE                                                                                       
hbase_score            

hive映射hbase表：
建立hive的外部表，映射HBase当中的表以及字段
在hive当中建立外部表，
进入hive客户端，然后执行以下命令进行创建hive外部表，就可以实现映射HBase当中的表数据

CREATE external TABLE course.hbase2hive(id int, name string, score int) STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler' WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key,cf:name,cf:score") TBLPROPERTIES("hbase.table.name" ="hbase_hive_score");