hbase分页代码实现完整版,包括建表,生成数据,分页

如果觉得代码格式乱可以去百度云下载地址:http://pan.baidu.com/s/1kTuVbwz

或者新浪微博:http://weibo.com/youlingR

建表和生成数据:

package youling.studio.page;


import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;


import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;


public class HbaseUtils {
private static Configuration conf = null;
private static HBaseConfiguration hconf = null;
private static HBaseAdmin admin = null;
static{
conf = new Configuration();
conf.set("hbase.zookeeper.quorum", "master,slave1,slave2");
conf.set("hbase.zookeeper.property.clientPort", "2181");
hconf = new HBaseConfiguration(conf);
try {
admin = new HBaseAdmin(hconf);
} catch (Exception e) {
e.printStackTrace();
}
}
public static void main(String[] args) throws Exception {
HbaseUtils hu = new HbaseUtils();
//hu.testCreateTable();
//hu.testDelTable();

hu.testPut();


}

/**
* 插入数据
* @throws Exception
*/
public void testPut() throws Exception{
String tableName = "students";
HTable table = new HTable(hconf, tableName);
table.setAutoFlush(false);
List<Put> records = new ArrayList<Put>();
int count = 100000;//10W
Random r = new Random();
for(int i = 0;i<count;i++){
Put p = new Put(String.format("row%09d", i).getBytes());
//Put p = new Put(String.format(""+i).getBytes());
p.add("info".getBytes(), "name".getBytes(), String.format("name%09d", i).getBytes());
p.add("info".getBytes(), "age".getBytes(), (r.nextInt(25)+"").getBytes());

p.add("course".getBytes(), "math".getBytes(), ((r.nextInt(40)+60)+"").getBytes());
p.add("course".getBytes(), "english".getBytes(), ((r.nextInt(40)+60)+"").getBytes());
p.add("course".getBytes(), "chinese".getBytes(), ((r.nextInt(40)+60)+"").getBytes());

records.add(p);
if(i%1000==0){
table.put(records);
records.clear();
}
}

table.flushCommits();
table.close();

}

/**
* 删除表
* @throws Exception
*/
public void testDelTable() throws Exception{
String tableName = "students";
if(admin.tableExists(tableName)){
admin.disableTable(tableName);
admin.deleteTable(tableName);
}
}

/**
* 创建表
* @throws Exception 
*/
public void testCreateTable() throws Exception{
HTableDescriptor htd = new HTableDescriptor("students");
htd.addFamily(new HColumnDescriptor("info"));
HColumnDescriptor hcd = new HColumnDescriptor("course");
hcd.setMaxVersions(10);//指定数据最大保存的版本个数。默认为3。
htd.addFamily(hcd);
htd.setMaxFileSize(64*1024*1024);//指定最大的region size

admin.createTable(htd);
}

}

分页代码:

package youling.studio.page2;


import java.io.IOException;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;


import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.HTablePool;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter;
import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
import org.apache.hadoop.hbase.util.Bytes;


public class Page {
private static Configuration config = null;
private static HTablePool tp = null;
static {
// 加载集群配置
config = HBaseConfiguration.create();
config.set("hbase.zookeeper.quorum", "master,slave1,slave2");//zk集群的地址,我的是三个节点的
config.set("hbase.zookeeper.property.clientPort", "2181");//zk的客户端的端口
// 创建表池(可伟略提高查询性能，具体说明请百度或官方API)
tp = new HTablePool(config, 10);//初始化hbase连接池的大小
}


public static void main(String[] args) throws IOException {
// 拿出row key的起始行和结束行//rowkey的设计必须用string设计,用long设计导致分布不均匀
// #<0<9<:
String startRow = "row000000020";//"row#"表示从第一条记录开始,"row000000020"表示从20开始包含开头
String stopRow = "row000000100";//"row:"表示到表的结尾,"row000000100"表示到到100条,这样的话就把扫描的rowkey限制在20-100的范围内,去掉多余的扫描工作
int currentPage = 1;
int pageSize = 20;
// 执行hbase查询
TBData tbData = getDataMap("students", startRow, stopRow, currentPage, pageSize);

for (Map<String, String> map : tbData.getResultList()) {
System.out.println(map);
}
System.out.println("aaa");
}
/**
* 查询数据
* @param tableKey 表标识
* @param queryKey 查询标识
* @param startRow 开始行
* @param paramsMap 参数集合
* @return 结果集
*/
public static TBData getDataMap(String tableName, String startRow,
String stopRow, Integer currentPage, Integer pageSize)
throws IOException {
List<Map<String, String>> mapList = null;
mapList = new LinkedList<Map<String, String>>();


ResultScanner scanner = null;
// 为分页创建的封装类对象，下面有给出具体属性
TBData tbData = null;
try {
// 获取最大返回结果数量
if (pageSize == null || pageSize == 0L)
pageSize = 100;


if (currentPage == null || currentPage == 0)
currentPage = 1;


// 计算起始页和结束页
Integer firstPage = (currentPage - 1) * pageSize;


Integer endPage = firstPage + pageSize;


// 从表池中取出HBASE表对象
HTableInterface table = getTable(tableName);
// 获取筛选对象
Scan scan = getScan(startRow, stopRow);
// 给筛选对象放入过滤器(true标识分页,具体方法在下面)
//scan.setFilter(packageFilters(true));

// 缓存1000条数据
scan.setCaching(1000);
scan.setCacheBlocks(false);
scanner = table.getScanner(scan);
int i = 0;
List<byte[]> rowList = new LinkedList<byte[]>();
// 遍历扫描器对象， 并将需要查询出来的数据row key取出
for (Result result : scanner) {
String row = toStr(result.getRow());
if (i >= firstPage && i < endPage) {
rowList.add(getBytes(row));
}
i++;
}


// 获取取出的row key的GET对象
List<Get> getList = getList(rowList);
Result[] results = table.get(getList);
// 遍历结果
for (Result result : results) {
Map<byte[], byte[]> fmap = packFamilyMap(result);
Map<String, String> rmap = packRowMap(fmap);
mapList.add(rmap);
}


// 封装分页对象
tbData = new TBData();
tbData.setCurrentPage(currentPage);
tbData.setPageSize(pageSize);
tbData.setTotalCount(i);
tbData.setTotalPage(getTotalPage(pageSize, i));
tbData.setResultList(mapList);
} catch (IOException e) {
e.printStackTrace();
} finally {
closeScanner(scanner);
}


return tbData;
}
/*
* 获取hbase的表
*/
public static HTableInterface getTable(String tableName) {


if (StringUtils.isEmpty(tableName))
return null;


return tp.getTable(getBytes(tableName));
}


/* 转换byte数组 */
public static byte[] getBytes(String str) {
if (str == null)
str = "";


return Bytes.toBytes(str);
}

/**
* 得到总页数
* @param pageSize
* @param totalCount
* @return
*/
private static int getTotalPage(int pageSize, int totalCount) {
int n = totalCount / pageSize;
if (totalCount % pageSize == 0) {
return n;
} else {
return ((int) n) + 1;
}
}


// 获取扫描器对象
private static Scan getScan(String startRow, String stopRow) {
Scan scan = new Scan();
scan.setStartRow(getBytes(startRow));
scan.setStopRow(getBytes(stopRow));


return scan;
}


/**
* 封装查询条件
*/
private static FilterList packageFilters(boolean isPage) {
FilterList filterList = null;
// MUST_PASS_ALL(条件 AND) MUST_PASS_ONE（条件OR）
filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
Filter filter1 = null;
Filter filter2 = null;
filter1 = newFilter(getBytes("info"), getBytes("age"),
CompareOp.EQUAL, getBytes("22"));
filter2 = newFilter(getBytes("course"), getBytes("math"),
CompareOp.LESS, getBytes("90"));
filterList.addFilter(filter1);
filterList.addFilter(filter2);
if (isPage) {
filterList.addFilter(new FirstKeyOnlyFilter());
}
return filterList;
}


private static Filter newFilter(byte[] f, byte[] c, CompareOp op, byte[] v) {
return new SingleColumnValueFilter(f, c, op, v);
}


private static void closeScanner(ResultScanner scanner) {
if (scanner != null)
scanner.close();
}


/**
* 封装每行数据
*/
private static Map<String, String> packRowMap(Map<byte[], byte[]> dataMap) {
Map<String, String> map = new LinkedHashMap<String, String>();


for (byte[] key : dataMap.keySet()) {


byte[] value = dataMap.get(key);


map.put(toStr(key), toStr(value));


}
return map;
}


/* 根据ROW KEY集合获取GET对象集合 */
private static List<Get> getList(List<byte[]> rowList) {
List<Get> list = new LinkedList<Get>();
for (byte[] row : rowList) {
Get get = new Get(row);


get.addColumn(getBytes("info"), getBytes("name"));
get.addColumn(getBytes("course"), getBytes("math"));
get.addColumn(getBytes("course"), getBytes("english"));
list.add(get);
}
return list;
}


/**
* 封装配置的所有字段列族
*/
private static Map<byte[], byte[]> packFamilyMap(Result result) {
Map<byte[], byte[]> dataMap = null;
dataMap = new LinkedHashMap<byte[], byte[]>();
dataMap.putAll(result.getFamilyMap(getBytes("info")));
dataMap.putAll(result.getFamilyMap(getBytes("course")));
return dataMap;
}


private static String toStr(byte[] bt) {
return Bytes.toString(bt);
}


}


//pagebean
class TBData {
private Integer currentPage;
private Integer pageSize;
private Integer totalCount;
private Integer totalPage;
private List<Map<String, String>> resultList;


public Integer getCurrentPage() {
return currentPage;
}


public void setCurrentPage(Integer currentPage) {
this.currentPage = currentPage;
}


public Integer getPageSize() {
return pageSize;
}


public void setPageSize(Integer pageSize) {
this.pageSize = pageSize;
}


public Integer getTotalCount() {
return totalCount;
}


public void setTotalCount(Integer totalCount) {
this.totalCount = totalCount;
}


public Integer getTotalPage() {
return totalPage;
}


public void setTotalPage(Integer totalPage) {
this.totalPage = totalPage;
}


public List<Map<String, String>> getResultList() {
return resultList;
}


public void setResultList(List<Map<String, String>> resultList) {
this.resultList = resultList;
}
}