Hbase源码研究（六）------put（4）

   今天来看一下HRegion类里面的doMiniBatchPut 方法，又是一个麻烦的类........

@SuppressWarnings("unchecked")
  private long doMiniBatchPut(
      BatchOperationInProgress<Pair<Put, Integer>> batchOp) throws IOException {

    WALEdit walEdit = new WALEdit();
    /* Run coprocessor pre hook outside of locks to avoid deadlock */
    if (coprocessorHost != null) {
      for (int i = 0; i < batchOp.operations.length; i++) {
        Pair<Put, Integer> nextPair = batchOp.operations[i];
        Put put = nextPair.getFirst();
        if (coprocessorHost.prePut(put, walEdit, put.getWriteToWAL())) {
          // pre hook says skip this Put
          // mark as success and skip below
          batchOp.retCodeDetails[i] = OperationStatus.SUCCESS;
        }
      }
    }

    long now = EnvironmentEdgeManager.currentTimeMillis();
    byte[] byteNow = Bytes.toBytes(now);
    boolean locked = false;

    /** Keep track of the locks we hold so we can release them in finally clause */
    List<Integer> acquiredLocks = Lists.newArrayListWithCapacity(batchOp.operations.length);
    // reference family maps directly so coprocessors can mutate them if desired
    Map<byte[],List<KeyValue>>[] familyMaps = new Map[batchOp.operations.length];
    // We try to set up a batch in the range [firstIndex,lastIndexExclusive)
    int firstIndex = batchOp.nextIndexToProcess;
    int lastIndexExclusive = firstIndex;
    boolean success = false;
    try {
      // ------------------------------------
      // STEP 1. Try to acquire as many locks as we can, and ensure
      // we acquire at least one.
      // ----------------------------------
      int numReadyToWrite = 0;
      while (lastIndexExclusive < batchOp.operations.length) {
        Pair<Put, Integer> nextPair = batchOp.operations[lastIndexExclusive];
        Put put = nextPair.getFirst();
        Integer providedLockId = nextPair.getSecond();

        Map<byte[], List<KeyValue>> familyMap = put.getFamilyMap();
        // store the family map reference to allow for mutations
        familyMaps[lastIndexExclusive] = familyMap;

        // skip anything that "ran" already
        if (batchOp.retCodeDetails[lastIndexExclusive].getOperationStatusCode()
            != OperationStatusCode.NOT_RUN) {
          lastIndexExclusive++;
          continue;
        }

        // Check the families in the put. If bad, skip this one.
        try {
          checkFamilies(familyMap.keySet());
        } catch (NoSuchColumnFamilyException nscf) {
          LOG.warn("No such column family in batch put", nscf);
          batchOp.retCodeDetails[lastIndexExclusive] = new OperationStatus(
              OperationStatusCode.BAD_FAMILY, nscf.getMessage());
          lastIndexExclusive++;
          continue;
        }

        // If we haven't got any rows in our batch, we should block to
        // get the next one.
        boolean shouldBlock = numReadyToWrite == 0;
        Integer acquiredLockId = getLock(providedLockId, put.getRow(), shouldBlock);
        if (acquiredLockId == null) {
          // We failed to grab another lock
          assert !shouldBlock : "Should never fail to get lock when blocking";
          break; // stop acquiring more rows for this batch
        }
        if (providedLockId == null) {
          acquiredLocks.add(acquiredLockId);
        }
        lastIndexExclusive++;
        numReadyToWrite++;
      }
      // Nothing to put -- an exception in the above such as NoSuchColumnFamily?
      if (numReadyToWrite <= 0) return 0L;

      // We've now grabbed as many puts off the list as we can

      // ------------------------------------
      // STEP 2. Update any LATEST_TIMESTAMP timestamps
      // ----------------------------------
      for (int i = firstIndex; i < lastIndexExclusive; i++) {
        // skip invalid
        if (batchOp.retCodeDetails[i].getOperationStatusCode()
            != OperationStatusCode.NOT_RUN) continue;

        updateKVTimestamps(
            familyMaps[i].values(),
            byteNow);
      }


      this.updatesLock.readLock().lock();
      locked = true;

      // ------------------------------------
      // STEP 3. Write to WAL
      // ----------------------------------
      for (int i = firstIndex; i < lastIndexExclusive; i++) {
        // Skip puts that were determined to be invalid during preprocessing
        if (batchOp.retCodeDetails[i].getOperationStatusCode()
            != OperationStatusCode.NOT_RUN) {
          continue;
        }

        Put p = batchOp.operations[i].getFirst();
        if (!p.getWriteToWAL()) continue;
        addFamilyMapToWALEdit(familyMaps[i], walEdit);
      }

      // Append the edit to WAL
      Put first = batchOp.operations[firstIndex].getFirst();
      this.log.append(regionInfo, this.htableDescriptor.getName(),
          walEdit, first.getClusterId(), now, this.htableDescriptor);

      // ------------------------------------
      // STEP 4. Write back to memstore
      // ----------------------------------
      long addedSize = 0;
      for (int i = firstIndex; i < lastIndexExclusive; i++) {
        if (batchOp.retCodeDetails[i].getOperationStatusCode()
            != OperationStatusCode.NOT_RUN) {
          continue;
        }
        addedSize += applyFamilyMapToMemstore(familyMaps[i]);
        batchOp.retCodeDetails[i] = OperationStatus.SUCCESS;
      }

      // ------------------------------------
      // STEP 5. Run coprocessor post hooks
      // ------------------------------------
      if (coprocessorHost != null) {
        for (int i = firstIndex; i < lastIndexExclusive; i++) {
          // only for successful puts
          if (batchOp.retCodeDetails[i].getOperationStatusCode()
              != OperationStatusCode.SUCCESS) {
            continue;
          }
          Put p = batchOp.operations[i].getFirst();
          coprocessorHost.postPut(p, walEdit, p.getWriteToWAL());
        }
      }

      success = true;
      return addedSize;
    } finally {
      if (locked)
        this.updatesLock.readLock().unlock();

      for (Integer toRelease : acquiredLocks) {
        releaseRowLock(toRelease);
      }
      if (!success) {
        for (int i = firstIndex; i < lastIndexExclusive; i++) {
          if (batchOp.retCodeDetails[i].getOperationStatusCode() == OperationStatusCode.NOT_RUN) {
            batchOp.retCodeDetails[i] = OperationStatus.FAILURE;
          }
        }
      }
      batchOp.nextIndexToProcess = lastIndexExclusive;
    }
  }

这个类看起来挺复杂的......没关系我们一步一步来分析。

先看传入参数 是一个自定义的类 BatchOperationInProgress

 private static class BatchOperationInProgress<T> {
    T[] operations;
    int nextIndexToProcess = 0;
    OperationStatus[] retCodeDetails;

    public BatchOperationInProgress(T[] operations) {
      this.operations = operations;
      this.retCodeDetails = new OperationStatus[operations.length];
      Arrays.fill(this.retCodeDetails, OperationStatus.NOT_RUN);
    }

    public boolean isDone() {
      return nextIndexToProcess == operations.length;
    }
  }

这个类 主要是封装了一下，Pair<Put, Integer>> ，并初始化了一下 返回值 OperationStatus[] ，都变成NOT_RUN。

首先在获得锁之前运行coprocessor ，以免发生死锁。

 /* Run coprocessor pre hook outside of locks to avoid deadlock */
    if (coprocessorHost != null) {
      for (int i = 0; i < batchOp.operations.length; i++) {
        Pair<Put, Integer> nextPair = batchOp.operations[i];
        Put put = nextPair.getFirst();
        if (coprocessorHost.prePut(put, walEdit, put.getWriteToWAL())) {
          // pre hook says skip this Put
          // mark as success and skip below
          batchOp.retCodeDetails[i] = OperationStatus.SUCCESS;
        }
      }
    }

对不需要做Put的Pair ， 直接把返回结果设置成SUCCESS ，可不参与后面的处理。

接下来获得系统时间，方便对ts进行插入

 long now = EnvironmentEdgeManager.currentTimeMillis();
    byte[] byteNow = Bytes.toBytes(now);
    boolean locked = false;

准备一个list用来记录获得的锁，准备一个map 以便后面的coprocessor可以直接修改。

    /** Keep track of the locks we hold so we can release them in finally clause */
    List<Integer> acquiredLocks = Lists.newArrayListWithCapacity(batchOp.operations.length);
    // reference family maps directly so coprocessors can mutate them if desired
    Map<byte[],List<KeyValue>>[] familyMaps = new Map[batchOp.operations.length];

还得补充一个准备工作，对于后面看代码有帮助，最后处理的是一个左闭右开的区间.......

// We try to set up a batch in the range [firstIndex,lastIndexExclusive)
    int firstIndex = batchOp.nextIndexToProcess;
    int lastIndexExclusive = firstIndex;

一切准备工作做的差不多了，正式进入提交环节。

先进行第一步，获得尽可能多的锁，并确保我们至少获得了一个，以便能进行提交。

      // ------------------------------------
      // STEP 1. Try to acquire as many locks as we can, and ensure
      // we acquire at least one.
      // ----------------------------------
      int numReadyToWrite = 0;
      while (lastIndexExclusive < batchOp.operations.length) {
        Pair<Put, Integer> nextPair = batchOp.operations[lastIndexExclusive];
        Put put = nextPair.getFirst();
        Integer providedLockId = nextPair.getSecond();

        Map<byte[], List<KeyValue>> familyMap = put.getFamilyMap();
        // store the family map reference to allow for mutations
        familyMaps[lastIndexExclusive] = familyMap;

        // skip anything that "ran" already
        if (batchOp.retCodeDetails[lastIndexExclusive].getOperationStatusCode()
            != OperationStatusCode.NOT_RUN) {
          lastIndexExclusive++;
          continue;
        }

        // Check the families in the put. If bad, skip this one.
        try {
          checkFamilies(familyMap.keySet());
        } catch (NoSuchColumnFamilyException nscf) {
          LOG.warn("No such column family in batch put", nscf);
          batchOp.retCodeDetails[lastIndexExclusive] = new OperationStatus(
              OperationStatusCode.BAD_FAMILY, nscf.getMessage());
          lastIndexExclusive++;
          continue;
        }

        // If we haven't got any rows in our batch, we should block to
        // get the next one.
        boolean shouldBlock = numReadyToWrite == 0;
        Integer acquiredLockId = getLock(providedLockId, put.getRow(), shouldBlock);
        if (acquiredLockId == null) {
          // We failed to grab another lock
          assert !shouldBlock : "Should never fail to get lock when blocking";
          break; // stop acquiring more rows for this batch
        }
        if (providedLockId == null) {
          acquiredLocks.add(acquiredLockId);
        }
        lastIndexExclusive++;
        numReadyToWrite++;
      }

把整个BatchOperationInProgress做一次循环，遍历出每一个Pair , 从每一个Pair里拿到put和对应的私有的lockid ， 跳过已经处理过的数据和familyMap不合法的数据。

通过

Integer acquiredLockId = getLock(providedLockId, put.getRow(), shouldBlock);

方法来获得此条put对应的锁，并把锁加入到刚才声明的acquiredLocks这个List里。

第一步操作到此就结束了，完成了他获得锁的使命，进入第二步---修改timestamp

// ------------------------------------
      // STEP 2. Update any LATEST_TIMESTAMP timestamps
      // ----------------------------------
      for (int i = firstIndex; i < lastIndexExclusive; i++) {
        // skip invalid
        if (batchOp.retCodeDetails[i].getOperationStatusCode()
            != OperationStatusCode.NOT_RUN) continue;

        updateKVTimestamps(
            familyMaps[i].values(),
            byteNow);
      }

这个很好理解，下面有个细节......关于hbase写的时候的锁问题，他用的是读写锁里的读锁。

this.updatesLock.readLock().lock();

第三步 往WAL里写数据，这个比较清楚.......

// ------------------------------------
      // STEP 3. Write to WAL
      // ----------------------------------
      for (int i = firstIndex; i < lastIndexExclusive; i++) {
        // Skip puts that were determined to be invalid during preprocessing
        if (batchOp.retCodeDetails[i].getOperationStatusCode()
            != OperationStatusCode.NOT_RUN) {
          continue;
        }

        Put p = batchOp.operations[i].getFirst();
        if (!p.getWriteToWAL()) continue;
        addFamilyMapToWALEdit(familyMaps[i], walEdit);
      }

      // Append the edit to WAL
      Put first = batchOp.operations[firstIndex].getFirst();
      this.log.append(regionInfo, this.htableDescriptor.getName(),
          walEdit, first.getClusterId(), now, this.htableDescriptor);

第四步 看起来就比较清晰了，往memstore里灌数据

 // ------------------------------------
      // STEP 4. Write back to memstore
      // ----------------------------------
      long addedSize = 0;
      for (int i = firstIndex; i < lastIndexExclusive; i++) {
        if (batchOp.retCodeDetails[i].getOperationStatusCode()
            != OperationStatusCode.NOT_RUN) {
          continue;
        }
        addedSize += applyFamilyMapToMemstore(familyMaps[i]);
        batchOp.retCodeDetails[i] = OperationStatus.SUCCESS;
      }

其中需要记录灌了多少数据 addedSize 并返回去.......

最后一步，运行coprocessor 进行收尾工作

// ------------------------------------
      // STEP 5. Run coprocessor post hooks
      // ------------------------------------
      if (coprocessorHost != null) {
        for (int i = firstIndex; i < lastIndexExclusive; i++) {
          // only for successful puts
          if (batchOp.retCodeDetails[i].getOperationStatusCode()
              != OperationStatusCode.SUCCESS) {
            continue;
          }
          Put p = batchOp.operations[i].getFirst();
          coprocessorHost.postPut(p, walEdit, p.getWriteToWAL());
        }
      }

着重再看下finally里做的处理

 finally {
      if (locked)
        this.updatesLock.readLock().unlock();

      for (Integer toRelease : acquiredLocks) {
        releaseRowLock(toRelease);
      }
      if (!success) {
        for (int i = firstIndex; i < lastIndexExclusive; i++) {
          if (batchOp.retCodeDetails[i].getOperationStatusCode() == OperationStatusCode.NOT_RUN) {
            batchOp.retCodeDetails[i] = OperationStatus.FAILURE;
          }
        }
      }
      batchOp.nextIndexToProcess = lastIndexExclusive;
    }

先把读写锁还了，再把获得的锁也还了，最后如果不成功（有异常没执行完5个步骤的话），把返回值置为FAILURE。最后把 batchOp.nextIndexToProcess 置成最后。

终于啰嗦完了，说的有点儿粗.........以后觉得哪有必要说细了，再细着说下吧......