hbase0.96+hadoop2.2分页中遇到的问题

最近项目中用到了hbase，开始写的查询效率非常慢，由于用到了

Filter startRowkeyFilter = new RowFilter(CompareFilter.CompareOp.EQUAL, new BinaryPrefixComparator(startRow.getBytes()));//rowkey前匹配

  Filter stopRowFilter = new RowFilter(CompareFilter.CompareOp.EQUAL,new RegexStringComparator(".*"+stopRow+"$"));//正则后匹配

new PrefixFilter(startRow.getBytes())

由于我们的数据量特别大导致哦相应速度到了不能忍受的慢

经过代码调优我们用到了

long luserId = Long.parseLong(uid);
    String strUserId = Long.toHexString(luserId);
   String startKey = mergeKey(strUserId, 8, "0");
    Long l = luserId + 1;
   String endKey =mergeKey(Long.toHexString(l), 8, "0");
     scan.setStartRow(startKey.getBytes());
     scan.setStopRow(endKey.getBytes());

用startrow与stoprow先去过滤

加上列的过滤

SingleColumnValueFilter gt = new SingleColumnValueFilter(
        family.getBytes(), filterRow.getBytes(),
     CompareFilter.CompareOp.GREATER_OR_EQUAL,Long.toString(startTime).getBytes());

效率由几十秒调到了几十毫秒

经过比对分析，hbase是的RowFilter和PrefixFilter是在所有filter过滤完后全表匹配rowkey效率非常低，所以要用StartRow与StopRow限定范围后再用RowFilter或者PrefixFilter进行过滤

hbase分页最好不要像我们的关系性数据库分页计算出总条数后做分页特别耗费资源

所以系统只提供下一页

如果知道上次查询的最后一条rowkey直接用

scan.setStartRow(Bytes.add(lastRow.getBytes(), POSTFIX));没必要再用PrefixFilter 和RowFilter

帖一段分页代码

 /**
    * 根据rowkey扫描一段范围
    * @param tableName 表名
    * @param startRow 开始的行健
    * @param filterRow  需要过滤的时间列
    * @param stopRow  结束的行健
    * @param family  列族
    *
    * **/
   public static   Map<String,Object> scanByStartAndStopRowPageQuery(String tableName,
     String startRow,String stopRow,String filterRow,Builder bean,String family
     )throws Exception{
  final byte[] POSTFIX = new byte[] { 0x00 };
  Map<String,Object> map=new LinkedHashMap<String, Object>();
  String lastRow="";
  if(bean.hasLastRowKey()){
   lastRow=bean.getLastRowKey();
  }
  String uid="";
  long startMethodTime=System.currentTimeMillis();
  long totalNums = 0;//总共多少条
  int pageSize=0;
  if(bean.hasPageSize()){
   pageSize=bean.getPageSize();
  }
  pageSize++;//取当前请求+1 判断有无下一页
  long startTime=0;
  long endTime=0;
  List<Filter> flList = new ArrayList<Filter>();
  List<Result> list=new ArrayList<Result>();
     HTable table=new HTable(config, tableName);
     Scan scan=new Scan();
     if(bean.hasUid()){
      uid=bean.getUid();
     }
     if(StringUtils.isNotEmpty(lastRow)){//如果有传lastRow
      scan.setStartRow(Bytes.add(lastRow.getBytes(), POSTFIX));
     }else{
      //Filter startRowkeyFilter = new RowFilter(CompareFilter.CompareOp.EQUAL, new BinaryPrefixComparator(startRow.getBytes()));
   flList.add(new PrefixFilter(startRow.getBytes()));
//   //根据rowkey生成规则 如果是TABLE_TRACE_ITEMS不用匹配startrow与StopRow
//   if(tableName.indexOf(LogConstant.TABLE_TRACE_ITEMS)<0){
    if(StringUtils.isNotEmpty(uid)){
        long luserId = Long.parseLong(uid);
        String strUserId = Long.toHexString(luserId);
     String startKey = mergeKey(strUserId, 8, "0");
     Long l = luserId + 1;
     String endKey =mergeKey(Long.toHexString(l), 8, "0");
        scan.setStartRow(startKey.getBytes());
     scan.setStopRow(endKey.getBytes());
    }
//   }
   
   
     }
     if(bean.hasUid()){
      SingleColumnValueFilter uidFilter= new SingleColumnValueFilter(
        family.getBytes(),"uid".getBytes(),CompareFilter.CompareOp.EQUAL,
        bean.getUid().getBytes());
      flList.add(uidFilter);
     }
     if(bean.hasSid()){
      SingleColumnValueFilter sidFilter= new SingleColumnValueFilter(
        family.getBytes(),"sid".getBytes(),CompareFilter.CompareOp.EQUAL,
        bean.getSid().getBytes());
      flList.add(sidFilter);
     }
     if(bean.hasDevid()){
      SingleColumnValueFilter devidFilter= new SingleColumnValueFilter(
        family.getBytes(),"devid".getBytes(),CompareFilter.CompareOp.EQUAL,
        bean.getDevid().getBytes());
      flList.add(devidFilter);
     }
     if(bean.hasSc()){//同步源过滤
      SingleColumnValueFilter scFilter= new SingleColumnValueFilter(
       family.getBytes(),"sc".getBytes(),CompareFilter.CompareOp.EQUAL,
       bean.getSc().getBytes());
      flList.add(scFilter);
     }
     if(bean.hasSvncmd()){//同步类型过滤
      SingleColumnValueFilter syncFilter= new SingleColumnValueFilter(
       family.getBytes(),"syncmd".getBytes(),CompareFilter.CompareOp.EQUAL,
       bean.getSvncmd().getBytes());
      flList.add(syncFilter);
     }

     if((bean.hasSt())){
      /*按开始结束时间过滤*/
      startTime=bean.getSt();
      
      SingleColumnValueFilter gt = new SingleColumnValueFilter(
        family.getBytes(), filterRow.getBytes(),
     CompareFilter.CompareOp.GREATER_OR_EQUAL,Long.toString(startTime).getBytes());
      
      flList.add(gt);
      
     }
     if(bean.hasEt()){
      endTime=bean.getEt();
      SingleColumnValueFilter et = new SingleColumnValueFilter(
        family.getBytes(), filterRow.getBytes(),
     CompareFilter.CompareOp.LESS_OR_EQUAL,Long.toString(endTime).getBytes());
      flList.add(et);
     }
     Filter filter = new PageFilter(pageSize);//分页filter
     flList.add(filter);
  
     scan.setFilter(new FilterList(Operator.MUST_PASS_ALL,flList));
     scan.setCaching(1000); 
  scan.setCacheBlocks(false);
     ResultScanner rs=table.getScanner(scan);
     pageSize--;
     if(rs!=null){
      for (Result r : rs) {
       totalNums++;
        if(totalNums<=pageSize){
         list.add(r);
        }
      }
  
     }
     logger.info("scanByStartAndStopRowPageQuery Table:"+tableName+";cost:"+(System.currentTimeMillis()-startMethodTime)+"ms；totalNums："+totalNums);
  rs.close();
     table.close();//释放资源
  map.put("totalNums", totalNums);
  map.put("result", list);
  return map;

 

上面就是对hbase分页浅谈