Hadoop学习笔记（四）HBase简单应用示例

HBase是一个建立在HDFS之上，面向列的针对结构化数据的可伸缩、高可靠、高性能、分布式和面向列的动态模式数据库。

HBase采用了BigTable的数据模型：增强的稀疏排序映射表（Key/Value），其中，键由行关键字、列关键字和时间戳构成。

HBase提供了对大规模数据的随机、实时读写访问，同时，HBase中保存的数据可以使用MapReduce来处理，它将数据存储和并行计算完美地结合在一起。

 

注意：搭建使用Hbase环境时，由于其依赖于Hadoop和Zookeeper，所以需要提前搭建并启动好Hadoop和ZooKeeper集群。

 

HBase使用示例：

pom.xml中添加：

        <dependency>
			<groupId>org.apache.hbase</groupId>
			<artifactId>hbase-client</artifactId>
			<version>1.1.2</version>
		</dependency>

HBaseTest .java

package com.hbase;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
import org.testng.annotations.AfterTest;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;

import com.common.FileUtil;
import com.common.TestBase;

public class HBaseTest extends TestBase{
	int iCount = 10;//写入读取数据数
	String fileName = "29591_九阳绝神/九阳绝神_0078.txt";

	TableName tableName = TableName.valueOf("testTable");
	FileUtil fileUtil = new FileUtil();

	public static Configuration configuration;
	static {
		// 取得一个数据库连接的配置参数对象
		configuration = HBaseConfiguration.create();
		// 设置连接参数：HBase数据库使用的端口
		configuration.set("hbase.zookeeper.property.clientPort", "2181");
		// 设置连接参数：HBase数据库所在的主机IP
		configuration.set("hbase.zookeeper.quorum", "172.16.1.22");
		// 集群主机，如有集群必须配置
		configuration.set("hbase.master", "172.16.1.22:16000");
	}

	// 与HBase数据库的连接对象
	Connection connection;
	// 数据库元数据操作对象
	Admin admin;

	@BeforeTest(description = "连接")
	public void before_test() throws IOException {
		System.out.println("-----连接-----");
		// 取得一个数据库连接对象
		connection = ConnectionFactory.createConnection(configuration);
		// 取得一个数据库元数据操作对象
		admin = connection.getAdmin();
		System.out.println("-----取得了数据库元数据操作对象-----");

		// 创建表
		// creatTable();
	}

	@Test(description = "创建表")
	public void creatTable() throws IOException {
		System.out.println("start create table ......");
		try {

			if (admin.tableExists(tableName)) {
				// 如果存在要创建的表，那么先删除，再创建
				admin.disableTable(tableName);
				admin.deleteTable(tableName);
				System.out.println(tableName + " is exist,detele....");
			}
			HTableDescriptor tableDescriptor = new HTableDescriptor(tableName);
			tableDescriptor.addFamily(new HColumnDescriptor("column"));
			// tableDescriptor.addFamily(new HColumnDescriptor("information"));
			// tableDescriptor.addFamily(new HColumnDescriptor("contact"));
			admin.createTable(tableDescriptor);
		} catch (Exception e) {
			e.printStackTrace();
		}
		System.out.println("end create table ......");
	}

	@Test(description = "插入数据")
	public void insert_test() throws IOException {
		String str = fileUtil.readFileToString(fileName);

		Long totalTime = Long.valueOf(0);
		for (int rowkey = 0; rowkey < iCount; rowkey++) {
			Put put = new Put(("user-" + rowkey).getBytes()); // ("user-" +
																// rowkey)是hbase存储中的rowkey
			// 参数：1.列族名 2.列名 3.值
			put.addColumn("column".getBytes(), "column1".getBytes(), String.valueOf(rowkey).getBytes());
			put.addColumn("column".getBytes(), "column2".getBytes(), String.valueOf(rowkey).getBytes());
			put.addColumn("column".getBytes(), "column3".getBytes(), str.getBytes());

			Table table = connection.getTable(tableName);

			long start_time = System.currentTimeMillis();
			table.put(put);
			long end_time = System.currentTimeMillis();
		//	System.out.println("第" + rowkey + "条写入时间" + (end_time - start_time) + "ms");
			logger.info("第" + rowkey + "条写入时间" + (end_time - start_time) + "ms");
			totalTime = totalTime + (end_time - start_time);
		}

		System.out.println("最终写入数据时间：" + totalTime + "ms");
	}

	@Test(description = "按条件查询数据")
	public void findOne_test() throws IOException, InterruptedException {

		Long totalTime = Long.valueOf(0);
		Long start_time = Long.valueOf(0);
		Long end_time = Long.valueOf(0);

		Table table = connection.getTable(tableName);

		
		for (int i = 0; i < iCount; i++) {

			String rowKey = "user-" + i;
			Get get = new Get(rowKey.getBytes());

			if (!get.isCheckExistenceOnly()) {
				start_time = System.currentTimeMillis();
				Result result = table.get(get);
				//end_time = System.currentTimeMillis();
				for (Cell cell : result.rawCells()) {
							;
					//String colName = Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(),
					//		cell.getQualifierLength());
					//String value = Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());

				}
				end_time = System.currentTimeMillis();
				//System.out.println("读取第" + i + "条数据时间" + (end_time - start_time) + "ms");
				logger.info("读取第" + i + "条数据时间" + (end_time - start_time) + "ms");
				totalTime = totalTime + (end_time - start_time);
			}
		}
		System.out.println("最终读取数据时间：" + totalTime + "ms");

	}

	@Test(description = "查询所有数据")
	public void findAll_test() throws IOException, InterruptedException {

		Long start_time = Long.valueOf(0);
		Long end_time = Long.valueOf(0);
		try {
			Table table = connection.getTable(tableName);
			Scan scan = new Scan();
			start_time = System.currentTimeMillis();
			ResultScanner resutScanner = table.getScanner(scan);//获取当前给定列族的scanner实例

			for (Result result : resutScanner) {
				for (Cell cell : result.rawCells()) {
					;
					//String colName = Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(),
					//		cell.getQualifierLength());
					//String value = Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
				}

			}
			end_time = System.currentTimeMillis();
			System.out.println("读取所有数据时间：" + (end_time - start_time) + "ms");
		} catch (IOException e) {
			e.printStackTrace();
		}

	}

	@Test(description = "清空表")
	public void truncateTable_test() throws IOException {
		System.out.println("---------------清空表 START-----------------");
		// 设置表状态为无效
		admin.disableTable(tableName);
		// 清空指定表的数据
		admin.truncateTable(tableName, true);
		System.out.println("---------------清空表 End-----------------");

	}

	@Test(description = "删表")
	public void deleteTable_test() throws IOException {
		System.out.println("---------------删除表 START-----------------");
		// 设置表状态为无效
		admin.disableTable(tableName);
		// 删除指定的数据表
		admin.deleteTable(tableName);
		System.out.println("---------------删除表 End-----------------");

	}

	@AfterTest(description = "结束测试")
	public void afterTest() throws IOException {
		connection.close();
		admin.close();
		System.out.println("---------------结束测试-----------------");

	}

}