hbase2.0 mob file compact 优化到regionserver上实现, 成倍提升性能

writer  伍增田

当前mob compact功能是在master上运行的, 性能大大受限, 可以把它分布到多台regionserver上执行,  极大地提高运行性能.   海量图片小文件的存储.

解决方案:

    1 master基本保持不变, 完成mob compact任务的建立, 包括选择文件, compactPartition收集, del file的compact, 最后按compactPartitionId调用rpc接口分发到多台regionserver上执行具体的mob compact file过程.

    

regionserver增加新的rpc接口,  Admin.proto

// mob compact on regionserver
message CompactionPartitionIdRS {
  string startKey = 1;
  string date = 2;
  string latestDate = 3;
  uint64 threshold = 4;    
}

message CompactionPartitionRS {
  repeated string files = 1;
  CompactionPartitionIdRS partitionId = 2;
  bytes startKey = 3;
  bytes endKey = 4;
}

message CompactMobFilePartitionRSRequest {
  uint64 selectionTime = 1;
  CompactionPartitionRS compactionPartitionRS = 2;
  repeated string delFiles = 3;
  bytes tablename = 4;
  bytes columnFamily = 5;
}

message CompactMobFilePartitionRSResponse {
  repeated string files = 1;
}

// newly added
  rpc CompactMobFilePartitionRS(CompactMobFilePartitionRSRequest)
    returns(CompactMobFilePartitionRSResponse);

新增加的代码如下:

PartitionedMobCompactor.java

private List<Path> callCompactMobFilePartitionRS(PartitionedMobCompactionRequest request,
          CompactionPartition partition,
          List<HStoreFile> delFiles,
          Connection connection,
          Table table) throws IOException {      
    CompactionPartitionIdRS.Builder cpIdBuilder = CompactionPartitionIdRS.newBuilder();
      cpIdBuilder.setStartKey(partition.getPartitionId().getStartKey())
                 .setDate(partition.getPartitionId().getDate())
                 .setLatestDate(partition.getPartitionId().getLatestDate())
                 .setThreshold(partition.getPartitionId().getThreshold());      
      CompactionPartitionIdRS cpIdRS = cpIdBuilder.build();
        
      CompactionPartitionRS.Builder cpBuilder = CompactionPartitionRS.newBuilder();
      for(FileStatus fst : partition.listFiles()) {
        cpBuilder.addFiles(fst.getPath().toString());
      }
      cpBuilder.setPartitionId(cpIdRS)
               .setStartKey(UnsafeByteOperations.unsafeWrap(partition.getStartKey()))
               .setEndKey(UnsafeByteOperations.unsafeWrap(partition.getEndKey()));
      CompactionPartitionRS cprs = cpBuilder.build();
        
      CompactMobFilePartitionRSRequest.Builder cmfpReqBuilder = CompactMobFilePartitionRSRequest.newBuilder();
      for(HStoreFile hsf : delFiles) {
          cmfpReqBuilder.addDelFiles(hsf.getPath().toString());    
      }
      cmfpReqBuilder.setSelectionTime(request.selectionTime)
                    .setCompactionPartitionIdRS(cprs)
                    .setTablename(UnsafeByteOperations.unsafeWrap(table.getName()))
                    .setColumnFamily(UnsafeByteOperations.unsafeWrap(this.column.getName()));
      CompactMobFilePartitionRSRequest cmfpReq = cmfpReqBuilder.build();
      
      HMaster master = (HMaster)server;
      List<Path> newFiles = new ArrayList<>();
      List<ServerName> serverNames = master.getServerManager().getOnlineServersList();
      ServerName serverName = serverNames.get(new Random().nextInt(serverNames.size()));
      final AdminService.BlockingInterface admin = master.getServerManager()
              .getRsAdmin(serverName);
      CompactMobFilePartitionRSResponse respondse;
      LOG.info("start Compaction " + partition.getPartitionId() + " on " +  serverName);
      try {
        respondse = admin.CompactMobFilePartitionRS(null, cmfpReq);
      }catch (ServiceException se) {
         throw ProtobufUtil.getRemoteException(se);
      }
    
      for(String fn : respondse.getFilesList()) {
        newFiles.add(new Path(fn));    
      }
      LOG.info(partition.getPartitionId() + " Compaction is finished. The number of mob files is changed from " + partition.listFiles().size()
          + " to " + newFiles.size());
      
    return newFiles;
  }
  
  public List<Path> compactMobFilePartitionRS(PartitionedMobCompactionRequest request,
          CompactionPartition partition,
          List<HStoreFile> delFiles,
          Connection connection,
          Table table) throws IOException {
    return compactMobFilePartition(request, partition, delFiles, connection, table);  
  }
  
  
  private Object server; // master or regionserver 
  public void setServer(Object server) {
    this.server = server;
  }
  
  public CacheConfig getCacheConfig() {
      return this.compactionCacheConfig;
  }
  
  public static CompactionPartitionId toCompactionPartitionId(CompactionPartitionIdRS partitionIdRS) {
      CompactionPartitionId partitionId = new CompactionPartitionId(partitionIdRS.getStartKey(), 
              partitionIdRS.getDate());
      partitionId.setLatestDate(partitionIdRS.getLatestDate());
      partitionId.setThreshold(partitionIdRS.getThreshold());
      return partitionId;
  }
  
  public static CompactionPartition toCompactionPartition(CompactionPartitionRS partitionRS, FileSystem fs) {
      CompactionPartition  partition = new CompactionPartition(
              toCompactionPartitionId(partitionRS.getPartitionId()));
      partition.setStartKey(partitionRS.getStartKey().toByteArray());
      partition.setEndKey(partitionRS.getEndKey().toByteArray());
      for(String file : partitionRS.getFilesList()) {
        partition.addFile(fs.getFileStatus(new Path(file)));
      }
      return partition;
  }  
RSRpcServices.java

@Override
  public CompactMobFilePartitionRSResponse CompactMobFilePartitionRS(RpcController controller,
     CompactMobFilePartitionRSRequest  request) throws ServiceException{
      try {  
        checkOpen();  
        long selectionTime = request.getSelectionTime();
        TableName tableName = TableName.valueOf(request.getTablename().toByteArray());
        final Connection c = ConnectionFactory.createConnection(this.regionServer.getConfiguration());
        ColumnFamilyDescriptor column = c.getAdmin().getDescriptor(tableName)
                .getColumnFamily(request.getColumnFamily().toByteArray());
             
        PartitionedMobCompactor compactor = null;
        compactor = new PartitionedMobCompactor(this.regionServer.getConfiguration(), 
                this.regionServer.getFileSystem(), tableName, column, null);
        compactor.setServer(this.regionServer);
        PartitionedMobCompactionRequest pmrequest = new PartitionedMobCompactionRequest(null, null);
        pmrequest.setSelectionTime(selectionTime);
        List<HStoreFile> delFiles = new ArrayList<>();
        for(String dfile : request.getDelFilesList()) {
          HStoreFile sf = new HStoreFile(this.regionServer.getFileSystem(), 
                        new Path(dfile), this.regionServer.getConfiguration(), 
                        compactor.getCacheConfig(), BloomType.NONE, true);
          delFiles.add(sf);
        }
      
      final Table table = c.getTable(tableName);
      CompactionPartitionId partitionId = PartitionedMobCompactor.toCompactionPartitionId(
              request.getCompactionPartitionRS().getCompactionPartitionIdRS());
      LOG.info("start Compaction " + partitionId + " " + tableName + ":" + column);
      List<Path> newFiles = compactor.compactMobFilePartitionRS(pmrequest,
              PartitionedMobCompactor.toCompactionPartition(request.getCompactionPartitionRS(), 
                  this.regionServer.getFileSystem()),
              delFiles,
              c,
              table);
      table.close();
      connection.close();
      LOG.info(partitionId + " " + tableName + ":" + column 
              + " Compaction is finished. The number of mob files is changed from " 
              + request.getCompactionPartitionRS().getFilesList().size()
              + " to " + newFiles.size());
      CompactMobFilePartitionRSResponse.Builder responseBld = CompactMobFilePartitionRSResponse.newBuilder();
      for(Path nfile : newFiles) {
        responseBld.addFiles(nfile.toString());
      }
      return responseBld.build();
      } catch (IOException ie) {
        throw new ServiceException(ie);
      }
  }

writer  伍增田