java hbase的scan分页_Hbase 分页设计

packagecp.app.service.impl;importjava.io.IOException;importjava.io.InterruptedIOException;importjava.util.ArrayList;importjava.util.HashMap;importjava.util.Iterator;importjava.util.List;importjava.util.Map;importjava.util.Map.Entry;importjava.util.Set;importorg.apache.commons.lang.StringUtils;importorg.apache.hadoop.conf.Configuration;importorg.apache.hadoop.hbase.Cell;importorg.apache.hadoop.hbase.HBaseConfiguration;importorg.apache.hadoop.hbase.HColumnDescriptor;importorg.apache.hadoop.hbase.HTableDescriptor;importorg.apache.hadoop.hbase.MasterNotRunningException;importorg.apache.hadoop.hbase.TableName;importorg.apache.hadoop.hbase.TableNotFoundException;importorg.apache.hadoop.hbase.ZooKeeperConnectionException;importorg.apache.hadoop.hbase.client.Get;importorg.apache.hadoop.hbase.client.HBaseAdmin;importorg.apache.hadoop.hbase.client.HConnection;importorg.apache.hadoop.hbase.client.HConnectionManager;importorg.apache.hadoop.hbase.client.HTable;importorg.apache.hadoop.hbase.client.HTableInterface;importorg.apache.hadoop.hbase.client.Put;importorg.apache.hadoop.hbase.client.Result;importorg.apache.hadoop.hbase.client.ResultScanner;importorg.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException;importorg.apache.hadoop.hbase.client.Scan;importorg.apache.hadoop.hbase.client.coprocessor.AggregationClient;importorg.apache.hadoop.hbase.client.coprocessor.LongColumnInterpreter;importorg.apache.hadoop.hbase.filter.BinaryComparator;importorg.apache.hadoop.hbase.filter.CompareFilter;importorg.apache.hadoop.hbase.filter.CompareFilter.CompareOp;importorg.apache.hadoop.hbase.filter.Filter;importorg.apache.hadoop.hbase.filter.FilterList;importorg.apache.hadoop.hbase.filter.PageFilter;importorg.apache.hadoop.hbase.filter.RowFilter;importorg.apache.hadoop.hbase.filter.SingleColumnValueFilter;importorg.apache.hadoop.hbase.filter.SubstringComparator;importorg.apache.hadoop.hbase.util.Bytes;importorg.apache.log4j.Logger;importorg.springframework.stereotype.Service;importcp.app.batch.utils.ConfigUtil;importcp.app.comm.CpConstants;importcp.app.service.HBaseService;/*** HBase查询与插入操作工具类

*

*@authorauthor

**/

//采用注入方式，HBaseService为定义的查询接口，可不需要。

@Servicepublic class HBaseServiceImpl implementsHBaseService{private static Logger log = Logger.getLogger(HBaseServiceImpl.class.getName());static ConfigUtil util = new ConfigUtil("conf/zookeeper.properties");private static final String HBASE_ZOOKEEPER_QUORUM =util

.getString("hbase_zookeeper_quorum");private static final String ZOOKEEPER_ZNODE_PARENT =util

.getString("zookeeper_znode_parent");private static Configuration conf =HBaseConfiguration.create();static{

conf.set("hbase.zookeeper.quorum", HBASE_ZOOKEEPER_QUORUM);

conf.set("zookeeper.znode.parent", ZOOKEEPER_ZNODE_PARENT);

}/*** 创建表

*

*@paramtableName

* 表名

*@paramcolumnFamily

* 列簇集合

*@return成功-true 失败-false*/@SuppressWarnings("resource")public boolean createTable(String tableName, ListcolumnFamily) {try{if (StringUtils.isBlank(tableName) || columnFamily == null

|| columnFamily.size() < 0) {

log.error("===Parameters tableName|columnFamily should not be null,Please check!===");

}

HBaseAdmin admin= newHBaseAdmin(conf);if(admin.tableExists(tableName)) {return true;

}else{

HTableDescriptor tableDescriptor= newHTableDescriptor(

TableName.valueOf(tableName));for(String cf : columnFamily) {

tableDescriptor.addFamily(newHColumnDescriptor(cf));

}

admin.createTable(tableDescriptor);

log.info("===Create Table " +tableName+ " Success!columnFamily:" +columnFamily.toString()+ "===");

}

}catch(MasterNotRunningException e) {//TODO Auto-generated catch block

log.error(e);return false;

}catch(ZooKeeperConnectionException e) {//TODO Auto-generated catch block

log.error(e);return false;

}catch(IOException e) {//TODO Auto-generated catch block

log.error(e);return false;

}return true;

}/*** 查询单条记录

*

*@paramtableName

* 表名

*@paramrowKey

* rowKey值

*@return返回单条记录*/

public List>selectOneByRowKey(String tableName,

String rowKey) {if (StringUtils.isBlank(rowKey) ||StringUtils.isBlank(tableName)) {

log.error("===Parameters tableName|rowKey should not be blank,Please check!===");return null;

}

List> rowList = new ArrayList>();try{

Get get= newGet(Bytes.toBytes(rowKey));

HTableInterface hTable=getHTable(tableName);if (hTable != null) {

Result result=hTable.get(get);

Map cellMap =getRowByResult(result);

rowList.add(cellMap);

}

hTable.close();

}catch(IOException e) {//TODO Auto-generated catch block

log.error(e);

}returnrowList;

}/*** 分页查询表数据

*

*@paramtableName

* 表名

*@paramddate

* 数据日期

*@parampageSize

* 页大小

*@paramlastrowKey

* 起始rowkey值

*@return返回查询数据结果集*/

public List>selectAllByPage(String tableName,

String ddate,intpageSize, String lastrowKey) {if (StringUtils.isBlank(tableName) ||StringUtils.isBlank(ddate)|| StringUtils.isBlank(pageSize + "")||StringUtils.isBlank(lastrowKey)) {

log.error("===Parameters tableName|ddate|pageSize|rowKey should not be blank,Please check!===");return null;

}

HTable hTable=(HTable) getHTable(tableName);

Scan scan= newScan();

FilterList filterList= newFilterList(

FilterList.Operator.MUST_PASS_ALL);

Filter rowFilter1= newRowFilter(CompareFilter.CompareOp.EQUAL,newSubstringComparator(ddate));

Filter pageFilter= newPageFilter(pageSize);

filterList.addFilter(rowFilter1);

filterList.addFilter(pageFilter);if (!CpConstants.ROWKEY_FIRST.equals(lastrowKey)) {

Filter rowFilter2= newRowFilter(CompareFilter.CompareOp.GREATER,newBinaryComparator(Bytes.toBytes(lastrowKey)));

filterList.addFilter(rowFilter2);

}

scan.setFilter(filterList);

List> lists = new ArrayList>();try{

ResultScanner rs=hTable.getScanner(scan);for(Result result : rs) {

lists.add(getRowByResult(result));

}

hTable.close();

}catch(IOException e) {//TODO Auto-generated catch block

log.error(e);

}returnlists;

}/*** 根据状态分页查询表数据

*

*@paramtableName

* 表名

*@paramddate

* 数据日期

*@parampageSize

* 页大小

*@paramlastrowKey

* 起始rowkey值

*@paramstatus

* 发送状态

*@return返回查询数据结果集*/

public List>selectAllByPageStatus(String tableName,

String ddate,intpageSize, String lastrowKey, String status) {if (StringUtils.isBlank(tableName) ||StringUtils.isBlank(ddate)|| StringUtils.isBlank(pageSize + "")||StringUtils.isBlank(lastrowKey)) {

log.error("===Parameters tableName|ddate|pageSize|rowKey should not be blank,Please check!===");return null;

}

HTable hTable=(HTable) getHTable(tableName);

Scan scan= newScan();

FilterList filterList= newFilterList(

FilterList.Operator.MUST_PASS_ALL);

filterList

.addFilter(new SingleColumnValueFilter(Bytes.toBytes("info"),

Bytes.toBytes("status"), CompareOp.EQUAL, Bytes

.toBytes(status)));

Filter rowFilter1= newRowFilter(CompareFilter.CompareOp.EQUAL,newSubstringComparator(ddate));

Filter pageFilter= newPageFilter(pageSize);

filterList.addFilter(rowFilter1);

filterList.addFilter(pageFilter);if (!CpConstants.ROWKEY_FIRST.equals(lastrowKey)) {

Filter rowFilter2= newRowFilter(CompareFilter.CompareOp.GREATER,newBinaryComparator(Bytes.toBytes(lastrowKey)));

filterList.addFilter(rowFilter2);

}

scan.setFilter(filterList);

List> lists = new ArrayList>();try{

ResultScanner rs=hTable.getScanner(scan);for(Result result : rs) {

lists.add(getRowByResult(result));

}

hTable.close();

}catch(IOException e) {//TODO Auto-generated catch block

log.error(e);

}returnlists;

}/*** 获取页数

*

*@paramtableName

* 表名

*@paramddate

* 数据日期

*@parampageSize

* 分页大小

*@return返回页数*/

public int getPages(String tableName, String ddate, intpageSize) {if (StringUtils.isBlank(tableName) ||StringUtils.isBlank(ddate)|| StringUtils.isBlank(pageSize + "")) {

log.error("===Parameters tableName|ddate|pageSize should not be blank,Please check!===");return 0;

}

enableAggregation(tableName);int total = 0;try{

HTable hTable=(HTable) getHTable(tableName);

Scan scan= newScan();

Filter rowFilter= newRowFilter(CompareFilter.CompareOp.EQUAL,newSubstringComparator(ddate));

scan.setFilter(rowFilter);

AggregationClient aggregation= newAggregationClient(conf);

Long count=aggregation.rowCount(hTable,newLongColumnInterpreter(), scan);

total=count.intValue();

hTable.close();

}catch(Throwable e) {//TODO Auto-generated catch block

log.error(e);

}return (total % pageSize == 0) ? total /pageSize

: (total/ pageSize) + 1;

}/*** 根据发送状态获取页数

*

*@paramtableName

* 表名

*@paramddate

* 数据日期

*@parampageSize

* 分页大小

*@paramstatus

* 发送状态

*@return返回页数*/

public int getPagesByStatus(String tableName, String ddate, intpageSize,

String status) {if (StringUtils.isBlank(tableName) ||StringUtils.isBlank(ddate)|| StringUtils.isBlank(pageSize + "")||StringUtils.isBlank(status)) {

log.error("===Parameters tableName|ddate|pageSize|status should not be blank,Please check!===");return 0;

}

enableAggregation(tableName);int total = 0;try{

HTable hTable=(HTable) getHTable(tableName);

Scan scan= newScan();

FilterList filterList= newFilterList(

FilterList.Operator.MUST_PASS_ALL);

Filter rowFilter= newRowFilter(CompareFilter.CompareOp.EQUAL,newSubstringComparator(ddate));

filterList.addFilter(rowFilter);

filterList.addFilter(newSingleColumnValueFilter(Bytes

.toBytes("info"), Bytes.toBytes("status"), CompareOp.EQUAL,

Bytes.toBytes(status)));

scan.setFilter(filterList);

AggregationClient aggregation= newAggregationClient(conf);

Long count=aggregation.rowCount(hTable,newLongColumnInterpreter(), scan);

total=count.intValue();

hTable.close();

}catch(Throwable e) {//TODO Auto-generated catch block

log.error(e);

}return (total % pageSize == 0) ? total /pageSize

: (total/ pageSize) + 1;

}/*** 获取同一个rowkey下的记录集合

*

*@paramresult

* 结果集

*@return

*/

private MapgetRowByResult(Result result) {if (result == null) {

log.error("===Parameter |result| should not be null,Please check!===");return null;

}

Map cellMap = new HashMap();for(Cell cell : result.listCells()) {

String rowkey=Bytes.toString(cell.getRowArray(),

cell.getRowOffset(), cell.getRowLength());

String cf=Bytes.toString(cell.getFamilyArray(),

cell.getFamilyOffset(), cell.getFamilyLength());

String qf=Bytes.toString(cell.getQualifierArray(),

cell.getQualifierOffset(), cell.getQualifierLength());

String value=Bytes.toString(cell.getValueArray(),

cell.getValueOffset(), cell.getValueLength());

cellMap.put(CpConstants.HBASE_TABLE_PROP_ROWKEY, rowkey);

cellMap.put(CpConstants.HBASE_TABLE_PROP_COLUMNFAMILY, cf);

cellMap.put(qf, value);

}returncellMap;

}/*** 获取HTableInterface

*

*@paramtableName

* 表名

*@return返回HTableInterface实例*/

privateHTableInterface getHTable(String tableName) {if(StringUtils.isBlank(tableName)) {

log.error("===Parameter |tableName| should not be blank,Please check!===");return null;

}

HTableInterface hTable= null;try{

HConnection conn=HConnectionManager.createConnection(conf);

hTable=conn.getTable(Bytes.toBytes(tableName));

}catch(IOException e) {//TODO Auto-generated catch block

log.error(e);return null;

}returnhTable;

}/*** 批量插入或更新

*

*@paramtableName

* 表名

*@paramparaList

* 组装成json或xml后的参数

*@return成功-true 失败-false*/

public boolean batchPut(String tableName, List>paraList) {try{

List puts = new ArrayList();for (Mapmap : paraList) {

Put put=getPutByMap(map);

puts.add(put);

}

HTable hTable=(HTable) getHTable(tableName);

hTable.put(puts);

hTable.close();

}catch(RetriesExhaustedWithDetailsException e) {//TODO Auto-generated catch block

log.error(e);return false;

}catch(InterruptedIOException e) {//TODO Auto-generated catch block

log.error(e);return false;

}catch(IOException e) {//TODO Auto-generated catch block

log.error(e);return false;

}return true;

}/*** 根据map返回put

*

*@paramparaMap

* 参数map

*@return返回put*/

private Put getPutByMap(MapparaMap) {if (paraMap == null) {

log.error("===Parameter |paraMap| should not be null,Please check!===");return null;

}

Set> set =paraMap.entrySet();

Iterator> it =set.iterator();byte[] rowkey =Bytes.toBytes(paraMap

.get(CpConstants.HBASE_TABLE_PROP_ROWKEY));byte[] columnfamily =Bytes.toBytes(paraMap

.get(CpConstants.HBASE_TABLE_PROP_COLUMNFAMILY));

Put put= newPut(rowkey);while(it.hasNext()) {

Entry entry =it.next();

String key=entry.getKey();if (!CpConstants.HBASE_TABLE_PROP_ROWKEY.equals(key)&& !CpConstants.HBASE_TABLE_PROP_COLUMNFAMILY.equals(key)) {

String value=entry.getValue();

put.add(columnfamily, Bytes.toBytes(key), Bytes.toBytes(value));

}

}returnput;

}/*** 使表具有聚合功能

*

*@paramtableName

* 表名*/@SuppressWarnings("resource")private voidenableAggregation(String tableName) {

String coprocessorName= "org.apache.hadoop.hbase.coprocessor.AggregateImplementation";try{

HBaseAdmin admin= newHBaseAdmin(conf);

HTableDescriptor htd=admin.getTableDescriptor(Bytes

.toBytes(tableName));

List coprocessors =htd.getCoprocessors();if (coprocessors != null && coprocessors.size() > 0) {return;

}else{

admin.disableTable(tableName);

htd.addCoprocessor(coprocessorName);

admin.modifyTable(tableName, htd);

admin.enableTable(tableName);

}

}catch(TableNotFoundException e) {//TODO Auto-generated catch block

log.error(e);

}catch(MasterNotRunningException e) {//TODO Auto-generated catch block

log.error(e);

}catch(ZooKeeperConnectionException e) {//TODO Auto-generated catch block

log.error(e);

}catch(IOException e) {//TODO Auto-generated catch block

log.error(e);

}

}

}