hbase预分区建表

2019独角兽企业重金招聘Python工程师标准>>>

package cn.bsoft.hbase;

 

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.hbase.HBaseConfiguration;

import org.apache.hadoop.hbase.HColumnDescriptor;

import org.apache.hadoop.hbase.HTableDescriptor;

import org.apache.hadoop.hbase.ZooKeeperConnectionException;

import org.apache.hadoop.hbase.client.HBaseAdmin;

import org.apache.hadoop.hbase.client.HTable;

import org.apache.hadoop.hbase.client.HTableInterface;

import org.apache.hadoop.hbase.client.Put;

import org.apache.hadoop.hbase.util.Bytes;

 

import cn.bsoft.hbase.HBaseUtil;;

 

/**

*

*

* @author ZLH

* @time:2017年3月10日

*

*/

public class HBaseDAOImpl {

private static final Log logger = LogFactory.getLog(HBaseDAOImpl.class);

 

private HbaseTemplate htemplate;

 

public void setHtemplate(HbaseTemplate htemplate) {

this.htemplate = htemplate;

}

 

// 创建表及对应的列族信息

public void createHtable(String htablName, List<String> columenFamily) {

/** 协调器注册:hbase0.92之后新增加的一个高性能统计表行数的并行计算方式*/

public final static String coprocessClassName = "org.apache.hadoop.hbase.coprocessor.AggregateImplementation";

 

// cell 数据版本号控制 :主键rowkey相同的条件下，默认保留1000个版本

public static final int VERSION = 1000;

try {

Configuration conf = HBaseConfiguration.create();

conf.set("hbase.zookeeper.quorum", "hadoop01");

// HBaseAdmin hBaseAdmin = new HBaseAdmin(htemplate.getConfiguration());

HBaseAdmin hBaseAdmin = new HBaseAdmin(conf);

// 如果存在要创建的表，那么先删除，再创建

if (hBaseAdmin.tableExists(htablName)) {

/*try {

hBaseAdmin.disableTable(htablName);

} catch (Exception e) {

}

hBaseAdmin.deleteTable(htablName);*/

return;

}

byte [][] splitKeys =null;

while(true){

//随机取机数目 region数

HashChoreWoker worker = new HashChoreWoker(10000,6);

//产生splitkeys二维数组,这个数组存储的rowkey的边界值

splitKeys = worker.calcSplitKeys();

break;

}

//表描述

HTableDescriptor tableDescriptor = new HTableDescriptor(htablName);

// 注册协作器

tableDescriptor.addCoprocessor(coprocessClassName);

// 遍历columenFamily

for (String key : columenFamily) {

HColumnDescriptor newhcd = new HColumnDescriptor(key);

// CELL中保存数据的版本数

newhcd.setMaxVersions(VERSION);

tableDescriptor.addFamily(newhcd);

}

//hBaseAdmin.createTable(tableDescriptor);

//指定splitkeys 并创建表

hBaseAdmin.createTable(tableDescriptor,splitKeys);

} catch (MasterNotRunningException e) {

logger.error(e.toString());

} catch (ZooKeeperConnectionException e) {

logger.error(e.toString());

} catch (IOException e) {

logger.error(e.toString());

}

}

————————————————————————————————————

————————————————————————————————————

package cn.bsoft.hbase;

 

import java.util.Iterator;

import java.util.TreeSet;

import java.util.UUID;

 

import org.apache.hadoop.hbase.util.Bytes;

import org.apache.hadoop.hbase.util.MD5Hash;

 

/**

*

*

* @author ZLH

* @time:2017年3月10日

*

*/

public class HashChoreWoker {

//随机取机数目

private int baseRecord;

//rowkey生成器

// private RowKeyGenerator rkGen;

//取样时，由取样数目及region数相除所得的数量.

private int splitKeysBase;

//splitkeys个数

private int splitKeysNumber;

//由抽样计算出来的splitkeys结果

private byte[][] splitKeys;

public HashChoreWoker(int baseRecord, int prepareRegions) {

//随机取机数目

this.baseRecord = baseRecord;

// rkGen = new HashRowKeyGenerator();

//splitkeys个数

splitKeysNumber = prepareRegions - 1;

splitKeysBase = baseRecord / prepareRegions;

}

/**

* 获取二维数组边界值

* @return

*/

public byte[][] calcSplitKeys() {

//6-1=5splitKeysNumber

splitKeys = new byte[splitKeysNumber][];

/**

* 用treeset对rowkey进行排序，必须要对rowkey排序，

* 否则在调用admin.createTable(tableDescriptor,splitKeys)的时候会出错。

*/

TreeSet<byte[]> rows = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);

//随机取机数目10000

for (int i = 0; i < baseRecord; i++) {

//加入边界KEY

rows.add(nextId());

}

int pointer = 0;

//遍历边界rowkey

Iterator<byte[]> rowKeyIter = rows.iterator();

int index = 0;

while (rowKeyIter.hasNext()) {

byte[] tempRow = rowKeyIter.next();

rowKeyIter.remove();

//splitKeysBase取样值166

if ((pointer != 0) && (pointer % splitKeysBase == 0)) {

//index小于splitkeys个数

if (index < splitKeysNumber) {

splitKeys[index] = tempRow;

index ++;

}

}

pointer ++;

}

rows.clear();

rows = null;

return splitKeys;

}

/**

* 获取rowkey

* @return

*/

public static byte [] nextId(){

String eventid = getEventId();

byte [] rowkey = Bytes.add(MD5Hash.getMD5AsHex(Bytes.toBytes(eventid))

.substring(0, 8).getBytes(),Bytes.toBytes(eventid));

return rowkey;

}

/**

* 获取没有_的UUID

* @return

*/

public static String getEventId() {

String eventIdStr = UUID.randomUUID().toString();

String eventId = eventIdStr.substring(0, 8)

+ eventIdStr.substring(9, 13) + eventIdStr.substring(14, 18)

+ eventIdStr.substring(19, 23) + eventIdStr.substring(24);

return eventId;

}

}

转载于:https://my.oschina.net/zlhblogs/blog/902630