HBase

操作

shell

命名空间操作

# 创建一个普通的命名空间
hbase(main):006:0> create_namespace "zzj"

# 创建带属性的命名空间
hbase(main):007:0> create_namespace "shahe",{'subway'=>'bj'}

#删除命名空间
hbase(main):001:0> drop_namespace 'zzj'

# 描述命名空间
hbase(main):001:0> describe_namespace 'name'

# 修改命名空间
hbase(main):003:0> alter_namespace 'name',{METHOD => 'set', 'subway'=>'bj'}

# 列出所有命名空间
hbase(main):005:0> list_namespace

表操作

# 创建命名空间为zzj
hbase(main):007:0> create_namespace 'zzj'

# 创建zzj namespace的下的表叫做 t_user,有两个列簇叫做 cf1 cf2
hbase(main):008:0> create 'zzj:t_user','cf1','cf2'

# 直接删除无法删除,需要将其先disable掉
hbase(main):002:0> drop 'zzj:t_user'

# disable 表 (禁用表)
hbase(main):003:0> disable

# 删除表
hbase(main):004:0> drop 'zzj:t_user'

# 列出所有的表
hbase(main):002:0> list

# 列出某个命名空间下的表
hbase(main):003:0> list_namespace_tables 'zzj'

数据操作

  • put
# 最普通的插入语句的方式
hbase(main):005:0> put 'zzj:t_user','1','cf1:name','zhangsan'

# 将表对象复制给变量t
hbase(main):006:0> t = get_table 'zzj:t_user'

# 使用复制对象t 对表进行操作
hbase(main):007:0> t.put '1','cf1:age',18


# 覆盖操作和put插入相似 写入相同的条件  写入值就可覆盖
hbase(main):008:0> t.put '1','cf1:age',19

# 扫描 当前的表
hbase(main):009:0> t.scan
  • get
# 获取zzj:t_user 下的rowk为1 的所有的cell
hbase(main):013:0> get 'zzj:t_user' ,'1'

# 创建一个最多有三个版本的表
hbase(main):001:0> create 'zzj:t_user',{NAME=>'cf1',VERSIONS=>3}
  • delete /deleteall
hbase(main):009:0> t.delete '1','cf1:name'

# 根据时间戳进行删除
hbase(main):012:0> t.get '1',{COLUMN=>'cf1:age',VERSIONS=>3}

hbase(main):002:0> deleteall 'zzj:t_user','1'
  • scan
#  查询列簇为cf1 从第一行开始
hbase(main):019:0> scan 'zzj:t_user',{COLUMNS=>['cf1'],STARTROW=>'1'}

#  查询列簇为cf1 从第二行开始
hbase(main):020:0> scan 'zzj:t_user',{COLUMNS=>['cf1'],STARTROW=>'2'}


#  查询列簇为cf1 从第三行开始
hbase(main):021:0> scan 'zzj:t_user',{COLUMNS=>['cf1'],STARTROW=>'3'}


#  查询列簇为cf1 从第一行开始 只显示一行
hbase(main):022:0> scan 'zzj:t_user',{COLUMNS=>['cf1'],STARTROW=>'1',LIMIT=>1}

#  查询列簇为cf1 从第一行开始 只显示两行
hbase(main):023:0> scan 'zzj:t_user',{COLUMNS=>['cf1'],STARTROW=>'1',LIMIT=>2}
  • count
hbase(main):025:0> count 'zzj:t_user'
  • append
hbase(main):026:0> append 'zzj:t_user','1','cf1:name','110'
  • truncate 清空数据
hbase(main):028:0> truncate 'zzj:t_user'

Java

依赖

<dependency>
    <groupId>org.apache.hbase</groupId>
    <artifactId>hbase-client</artifactId>
    <version>1.2.4</version>
</dependency>

核心操作

	//获取客户端对象和连接对象
    @Before
    public void getClient() throws Exception {
        configuration = new Configuration();

        configuration.set("hbase.zookeeper.quorum", "HadoopNode00");
        configuration.set("hbase.zookeeper.property.clientPort", "2181");
        conn = ConnectionFactory.createConnection(configuration);
        admin = conn.getAdmin();

    }
    

    @After
    public void close() throws Exception {
        admin.close();
        conn.close();
    }

命名空间操作

 @Test
    public void createNameSpace() throws Exception {
        NamespaceDescriptor namespaceDescriptor = NamespaceDescriptor.create("zzj").addConfiguration("admin", "aaa").build();
        admin.createNamespace(namespaceDescriptor);
    }

    @Test
    public void listNameSpace() throws Exception {

        NamespaceDescriptor[] listNamespaceDescriptors = admin.listNamespaceDescriptors();

        for (NamespaceDescriptor listNamespaceDescriptor : listNamespaceDescriptors) {
            System.out.println(listNamespaceDescriptor.getName());
        }


    }

    @Test
    public void modifyNameSpace() throws Exception {
        NamespaceDescriptor namespaceDescriptor = NamespaceDescriptor.create("zzj").addConfiguration("aa", "bb").removeConfiguration("admin").build();
        admin.modifyNamespace(namespaceDescriptor);
    }

    @Test
    public void deleteNameSpace() throws Exception{
        admin.deleteNamespace("zzj");

    }

表常规

@Test
    public void createTables() throws Exception {

        /*
        创建表名的对象(封装表名字)
        * */
        TableName tableName = TableName.valueOf("zzj:t_user1");


        /*
         * 封装  表 的相关信息
         * */

        HTableDescriptor hTableDescriptor = new HTableDescriptor(tableName);

        /*
         * 封装列簇的相关信息
         * */
        HColumnDescriptor cf1 = new HColumnDescriptor("cf1");
        cf1.setMaxVersions(3);

= 
        /*
         *
         * */
        HColumnDescriptor cf2 = new HColumnDescriptor("cf2");
        cf2.setMaxVersions(3);

        /*
         * 在hTableDescriptor 对象中添加列簇描述对象
         * */


        hTableDescriptor.addFamily(cf1);
        hTableDescriptor.addFamily(cf2);


        /*
         * 创建 table
         * */
        admin.createTable(hTableDescriptor);

    }

    @Test
    public void dropTable() throws Exception {
        TableName tableName = TableName.valueOf("zzj:t_user1");
        admin.disableTable(tableName);
        admin.deleteTable(tableName);
    }

数据操作

  • put
@Test
    public void testPutOne() throws Exception {

        TableName tableName = TableName.valueOf("zzj:t_user");
        /*
         * 通过conn对象获得table的操作对象
         * */
        Table table = conn.getTable(tableName);

        Put put1 = new Put("1".getBytes());
        put1.addColumn("cf1".getBytes(), "name".getBytes(), "zhangsan".getBytes());
        put1.addColumn("cf1".getBytes(), "age".getBytes(), "18".getBytes());
        put1.addColumn("cf1".getBytes(), "sex".getBytes(), "false".getBytes());

       
        table.put(put1);
        table.close();
    }
 @Test
    public void testPutList() throws Exception {

        TableName tableName = TableName.valueOf("zzj:t_user");
        BufferedMutator bufferedMutator = conn.getBufferedMutator(tableName);


        Put put1 = new Put("4".getBytes());
        put1.addColumn("cf1".getBytes(), "name".getBytes(), "zhangsan".getBytes());
        put1.addColumn("cf1".getBytes(), "age".getBytes(), "18".getBytes());
        put1.addColumn("cf1".getBytes(), "sex".getBytes(), "false".getBytes());

        Put put2 = new Put("5".getBytes());
        put2.addColumn("cf1".getBytes(), "name".getBytes(), "zhangsan".getBytes());
        put2.addColumn("cf1".getBytes(), "age".getBytes(), "18".getBytes());
        put2.addColumn("cf1".getBytes(), "sex".getBytes(), "false".getBytes());

        Put put3 = new Put("6".getBytes());
        put3.addColumn("cf1".getBytes(), "name".getBytes(), "zhangsan".getBytes());
        put3.addColumn("cf1".getBytes(), "age".getBytes(), "18".getBytes());
        put3.addColumn("cf1".getBytes(), "sex".getBytes(), "false".getBytes());

        ArrayList<Put> puts = new ArrayList<Put>();
        puts.add(put1);
        puts.add(put2);
        puts.add(put3);

        bufferedMutator.mutate(puts);

        bufferedMutator.close();
    }

  • delete
    @Test
    public void tetsDelete() throws Exception {

        TableName tableName = TableName.valueOf("zzj:t_user");
        Table table = conn.getTable(tableName);
        Delete delete = new Delete("6".getBytes());

        table.delete(delete);
        table.close();
    }

    @Test
    public void testDeleteList() throws Exception{


        TableName tableName = TableName.valueOf("zzj:t_user");
        BufferedMutator bufferedMutator = conn.getBufferedMutator(tableName);

        Delete delete1 = new Delete("1".getBytes());
        Delete delete2 = new Delete("2".getBytes());
        Delete delete3 = new Delete("3".getBytes());

        ArrayList<Delete> deletes = new ArrayList<Delete>();

        deletes.add(delete1);
        deletes.add(delete2);
        deletes.add(delete3);

        bufferedMutator.mutate(deletes);

        bufferedMutator.close();


    }
  • get
  @Test
    public void testGet() throws Exception {

        TableName tableName = TableName.valueOf("zzj:t_user");
        Table table = conn.getTable(tableName);

        Get get = new Get("4".getBytes());

        Result result = table.get(get);

        byte[] name = result.getValue("cf1".getBytes(), "name".getBytes());
        byte[] age = result.getValue("cf1".getBytes(), "age".getBytes());
        byte[] sex = result.getValue("cf1".getBytes(), "sex".getBytes());

        System.out.println(new String(name) + "-" + new String(age) + "-" + new String(sex));

    }


    @Test
    public void testGet02() throws Exception {

        TableName tableName = TableName.valueOf("zzj:t_user");
        Table table = conn.getTable(tableName);

        Get get = new Get("4".getBytes());
        get.setMaxVersions(2);

        get.addColumn("cf1".getBytes(), "name".getBytes());

        Result result = table.get(get);

        List<Cell> cellList = result.getColumnCells("cf1".getBytes(), "name".getBytes());


        for (Cell cell : cellList) {
            /*
             * rowkey 列名  列值  时间戳
             * */
            byte[] rowkey = CellUtil.cloneRow(cell);
            byte[] cf = CellUtil.cloneFamily(cell);
            byte[] qualifier = CellUtil.cloneQualifier(cell);
            byte[] value = CellUtil.cloneValue(cell);
            long timestamp = cell.getTimestamp();

            System.out.println(new String(rowkey) + "--" + new String(cf) + "--" + new String(qualifier) + "--" + new String(value) +"--" +timestamp);
        }
  • scan
 @Test
    public void testScan() throws Exception {

        TableName tableName = TableName.valueOf("zzj:t_user");
        Table table = conn.getTable(tableName);

        Scan scan = new Scan();


        PrefixFilter prefixFilter1 = new PrefixFilter("4".getBytes());
        PrefixFilter prefixFilter2 = new PrefixFilter("5".getBytes());

        FilterList list = new FilterList(FilterList.Operator.MUST_PASS_ONE,prefixFilter1, prefixFilter2);
        scan.setFilter(list);


        ResultScanner results = table.getScanner(scan);

        for (Result result : results) {

            byte[] row = result.getRow();
            byte[] name = result.getValue("cf1".getBytes(), "name".getBytes());
            byte[] age = result.getValue("cf1".getBytes(), "age".getBytes());
            byte[] sex = result.getValue("cf1".getBytes(), "sex".getBytes());

            System.out.println(new String(row) + "--" + new String(name) + "-" + new String(age) + "-" + new String(sex));
        }
    }

MapReduce

依赖

<dependency>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-hdfs</artifactId>
    <version>2.6.0</version>
</dependency>
<dependency>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-common</artifactId>
    <version>2.6.0</version>
</dependency>

<dependency>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-mapreduce-client-core</artifactId>
    <version>2.6.0</version>
</dependency>
<dependency>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
    <version>2.6.0</version>
</dependency>

<dependency>
    <groupId>org.apache.hbase</groupId>
    <artifactId>hbase-client</artifactId>
    <version>1.2.4</version>
</dependency>
<dependency>
    <groupId>org.apache.hbase</groupId>
    <artifactId>hbase-server</artifactId>
    <version>1.2.4</version>
</dependency>
public class JobRunner {
    public static void main(String[] args) throws Exception {
        System.setProperty("HADOOP_USER_NAME", "root");

        Configuration configuration = new Configuration();

        configuration.set("hbase.zookeeper.quorum", "HadoopNode00");
        configuration.set("hbase.zookeeper.property.clientPort", "2181");

        configuration.addResource("conf2/core-site.xml");
        configuration.addResource("conf2/hdfs-site.xml");
        configuration.addResource("conf2/mapred-site.xml");
        configuration.addResource("conf2/yarn-site.xml");
        configuration.set(MRJobConfig.JAR, "G:\\IDEA_WorkSpace\\BigData\\HBase_Test\\target\\HBase_Test-1.0-SNAPSHOT.jar");
        configuration.set("mapreduce.app-submission.cross-platform", "true");

        Job job = Job.getInstance(configuration);
        job.setJarByClass(JobRunner.class);

        /*
         *
         * */
        job.setInputFormatClass(TableInputFormat.class);
        job.setOutputFormatClass(TableOutputFormat.class);

        /*
         *
         * 设置mapper 相关
         * */
        TableMapReduceUtil.initTableMapperJob(
                "zzj:t_user",
                new Scan(),
                UserMapper.class,
                Text.class,
                DoubleWritable.class,
                job
        );


        TableMapReduceUtil.initTableReducerJob(
                "zzj:t_result",
                UserReducer.class,
                job);

        job.waitForCompletion(true);
    }
}
public class UserMapper extends TableMapper<Text, DoubleWritable> {
    @Override
    protected void map(ImmutableBytesWritable key, Result value, Context context) throws IOException, InterruptedException {


        byte[] bytes = key.get();
        String rowkey = Bytes.toString(bytes);

        String company = rowkey.split(":")[0];

        byte[] salaryByte = value.getValue("cf1".getBytes(), "salary".getBytes());

        double salary = Bytes.toDouble(salaryByte);


        context.write(new Text(company), new DoubleWritable(salary));
    }
}
public class UserReducer extends TableReducer<Text, DoubleWritable, NullWritable> {

    @Override
    protected void reduce(Text key, Iterable<DoubleWritable> values, Context context) throws IOException, InterruptedException {
        double totalSalary = 0.0;
        int count = 0;

        for (DoubleWritable value : values) {
            totalSalary += value.get();
            count++;
        }

        Put put = new Put(key.getBytes());
        put.addColumn("cf1".getBytes(), "avgSalary".getBytes(), (totalSalary / count + "").getBytes());

        context.write(NullWritable.get(), put);
    }
}

搭建环境

环境

  • Hadoop HADOOP_HOME
  • Zookeeper

单机

[root@HadoopNode00 ~]# mkdir /usr/hbase
[root@HadoopNode00 ~]# tar -zxvf hbase-1.2.4-bin.tar.gz -C /usr/hbase/
[root@HadoopNode00 ~]# vi .bashrc
export HBASE_HOME=/usr/hbase/hbase-1.2.4
export HBASE_MANAGES_ZK=false
export PATH=$PATH:$HBASE_HOME/bin

[root@HadoopNode00 ~]# vi /usr/hbase/hbase-1.2.4/conf/hbase-site.xml

<property>
     <name>hbase.rootdir</name>
     <value>hdfs://HadoopNode00:9000/hbase</value>
</property>

<property>
	<name>hbase.zookeeper.quorum</name>
	<value>HadoopNode00</value>
</property>

<property>
	<name>hbase.zookeeper.property.clientPort</name>
	<value>2181</value>
</property>

<property>
        <name>hbase.cluster.distributed</name>
        <value>true</value>
</property>
[root@HadoopNode00 ~]# vi /usr/hbase/hbase-1.2.4/conf/regionservers
HadoopNode00

# 保证zk和hdfs启动成功
[root@HadoopNode00 zookeeper-3.4.6]# start-hbase.sh

集群

HadoopNode01HadoopNode02HadoopNode03
nn1nn2
journal nodejournal nodejournal node
zkfczkfc
datanodedatanodedatanode
zk01zk02zk03
rm1rm2
nodemanagernodemanagernodemanager
hregionserverhregionserverhregionserver
hmasterhmasterhmaster
# 首先在三台zk节点中保证ZK成功启动

[root@ZK01 ~]# /usr/zk/zookeeper-3.4.6/bin/zkServer.sh start /usr/zk/zookeeper-3.4.6/conf/zk.conf
JMX enabled by default
Using config: /usr/zk/zookeeper-3.4.6/conf/zk.conf
Starting zookeeper ... STARTED
[root@ZK01 ~]# /usr/zk/zookeeper-3.4.6/bin/zkServer.sh status /usr/zk/zookeeper-3.4.6/conf/zk.conf
JMX enabled by default
Using config: /usr/zk/zookeeper-3.4.6/conf/zk.conf
Mode: follower


# 配置安装Hbase 
[root@HadoopNodeX ~]# mkdir /usr/hbase
[root@HadoopNodeX ~]# tar -zxvf hbase-1.2.4-bin.tar.gz -C /usr/hbase/

# 配置环境比变量

[root@HadoopNodeX ~]# vi .bashrc
export HBASE_HOME=/usr/hbase/hbase-1.2.4
export HBASE_MANAGES_ZK=false
export PATH=$PATH:$HBASE_HOME/bin
# 环境变量生效
[root@HadoopNodeX ~]# source .bashrc


[root@HadoopNodeX ~]# vi /usr/hbase/hbase-1.2.4/conf/hbase-site.xml

<property>
     <name>hbase.rootdir</name>
     <value>hdfs://mycluster/hbase</value>
</property>

<property>
	<name>hbase.zookeeper.quorum</name>
	<value>ZK01,ZK02,ZK03</value>
</property>

<property>
	<name>hbase.zookeeper.property.clientPort</name>
	<value>2181</value>
</property>

<property>
        <name>hbase.cluster.distributed</name>
        <value>true</value>
</property>
[root@HadoopNodeX ~]# vi /usr/hbase/hbase-1.2.4/conf/regionservers
hadoopNode01
hadoopNode02
hadoopNode03

[root@HadoopNodeX ~]# hbase-daemon.sh start master
[root@HadoopNodeX ~]# hbase-daemon.sh start regionserver
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值