The process of OutOfCoreCallable

Constructor of ServerData

At the constructor of ServerData, if USE_OUT_OF_CORE_GRAPH is set true. oocEngine is created, and partitionStore is wrapped using DiskBackedPartitionStore.

PartitionStore<I, V, E> inMemoryPartitionStore =
        new SimplePartitionStore<I, V, E>(conf, context);
    if (GiraphConstants.USE_OUT_OF_CORE_GRAPH.get(conf)) {
      oocEngine = new OutOfCoreEngine(conf, service, workerServer);
      partitionStore =
          new DiskBackedPartitionStore<I, V, E>(inMemoryPartitionStore,
              conf, context, oocEngine);
      edgeStore =
          new DiskBackedEdgeStore<I, V, E>(inMemoryEdgeStore, conf, oocEngine);
    }

Constructor of OutOfCoreEngine

public OutOfCoreEngine(ImmutableClassesGiraphConfiguration<?, ?, ?> conf,
                         CentralizedServiceWorker<?, ?, ?> service,
                         NetworkMetrics networkMetrics) {

dataAccessor in OutOfCoreEngine

dataAccessor is instance of LocalDiskDataAccessor.
dataAccessor set numIOThreads by PARTITIONS_DIRECTORY.

  StrConfOption PARTITIONS_DIRECTORY =
      new StrConfOption("giraph.partitionsDirectory", "_bsp/_partitions",
          "Comma-separated list of directories in the local filesystem for " +
          "out-of-core partitions.");

Class<? extends OutOfCoreDataAccessor> accessorClass =
        GiraphConstants.OUT_OF_CORE_DATA_ACCESSOR.get(conf);
    try {
      Constructor<?> constructor = accessorClass.getConstructor(
          ImmutableClassesGiraphConfiguration.class);
      this.dataAccessor = (OutOfCoreDataAccessor) constructor.newInstance(conf);
    } catch (NoSuchMethodException | InstantiationException |
        InvocationTargetException | IllegalAccessException e) {
      throw new IllegalStateException("OutOfCoreEngine: caught exception " +
          "while creating the data accessor instance!", e);
    }

create OutOfCoreIOCallableFactory

this.oocIOCallableFactory =
        new OutOfCoreIOCallableFactory(this, numIOThreads,
            service.getGraphTaskManager().createUncaughtExceptionHandler());

OutOfCoreEngine.initialize

OutOfCoreEngine.initialize call oocIOCallableFactory.createCallable();

public void initialize() {
    oocIOCallableFactory.createCallable();
  }

OutOfCoreIOCallableFactory.createCallable

OutOfCoreIOCallableFactory uses a ThreadPoolExecutor to submit Call

public void createCallable() {
    CallableFactory<Void> outOfCoreIOCallableFactory =
      new CallableFactory<Void>() {
        @Override
        public Callable<Void> newCallable(int callableId) {
          return new OutOfCoreIOCallable(oocEngine, callableId);
        }
      };
    outOfCoreIOExecutor = new ThreadPoolExecutor(numIOThreads, numIOThreads, 0L,
        TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>(),
        ThreadUtils.createThreadFactory("ooc-io-%d"));

    for (int i = 0; i < numIOThreads; ++i) {
      Future<Void> future = ThreadUtils.submitToExecutor(outOfCoreIOExecutor,
          outOfCoreIOCallableFactory.newCallable(i), uncaughtExceptionHandler);
      results.add(future);
    }
    // Notify executor to not accept any more tasks
    outOfCoreIOExecutor.shutdown();
  }

OutOfCoreIOCallable.call

OutOfCoreIOCallable.call acqure IOCommand from occEngine.OSScheduler

public Void call() throws Exception {
    while (true) {

      IOCommand command = oocEngine.getIOScheduler().getNextIOCommand(diskId);
        boolean commandExecuted = false;
       commandExecuted = command.execute();
      oocEngine.getIOScheduler().ioCommandCompleted(command);
  }
  return null;
  }

OutOfCoreIOScheduler.getNextIOCommand

  public IOCommand getNextIOCommand(int threadId) {
              IOCommand command = null;
              OutOfCoreOracle.IOAction[] actions =
          oocEngine.getOracle().getNextIOActions();

There are several type of OutOfCoreOracle, includes MemoryEstimatorOracle, ThresholdBasedOracle. FixedPartitionsOracle, SimpleGCMonitoringOracle

   int numPartitionsInMemory =
        oocEngine.getMetaPartitionManager().getNumInMemoryPartitions();
    int numPartialPartitionsInMemory =
        oocEngine.getMetaPartitionManager().getNumPartiallyInMemoryPartitions();
int numPartitions =
        numPartitionsInMemory + deltaNumPartitionsInMemory.get();
 if (numPartitions < maxPartitionsInMemory) {
      return new IOAction[]{
        IOAction.LOAD_PARTITION,
        IOAction.STORE_MESSAGES_AND_BUFFERS};
    } else if (numPartitions > maxPartitionsInMemory) {
      return new IOAction[]{
        IOAction.STORE_PARTITION,
        IOAction.STORE_MESSAGES_AND_BUFFERS};
    } else {
      return new IOAction[]{
        IOAction.STORE_MESSAGES_AND_BUFFERS,
        IOAction.LOAD_TO_SWAP_PARTITION};
    }

Continue OutOfCoreIOScheduler.getNextIOCommand

 // Check whether there are any urgent outstanding load requests
   if (!threadLoadCommandQueue.get(threadId).isEmpty()) {
        boolean canLoad = false;
        for (OutOfCoreOracle.IOAction action : actions) {
          if (action == OutOfCoreOracle.IOAction.LOAD_PARTITION ||
              action == OutOfCoreOracle.IOAction.LOAD_UNPROCESSED_PARTITION ||
              action == OutOfCoreOracle.IOAction.LOAD_TO_SWAP_PARTITION ||
              action == OutOfCoreOracle.IOAction.URGENT_LOAD_PARTITION) {
            canLoad = true;
            break;
          }
        }
        if (canLoad) {
          command = threadLoadCommandQueue.get(threadId).poll();
          checkNotNull(command);
          if (oocEngine.getOracle().approve(command)) {
            return command;
          } else {
            // Loading is not viable at this moment. We should put the command
            // back in the load queue and wait until loading becomes viable.
            threadLoadCommandQueue.get(threadId).offer(command);
          }
        }
      }

the element in threadLoadCommandQueue is added by OutOfCoreIOScheduler.addIOCommand

public void addIOCommand(IOCommand ioCommand) {
    if (ioCommand instanceof LoadPartitionIOCommand) {
      int ownerThread = oocEngine.getMetaPartitionManager()
          .getOwnerThreadId(ioCommand.getPartitionId());
      threadLoadCommandQueue.get(ownerThread).offer(ioCommand);
    } else {
      throw new IllegalStateException("addIOCommand: IO command type is not " +
          "supported for addition");
    }
  }

OutOfCoreEngine.retrievePartition

For example,OutOfCoreEngine.retrievePartition will wait until partition in memory.

/**
   * Retrieve a particular partition. After this method is complete the
   * requested partition should be in memory.
   *
   * @param partitionId id of the partition to retrieve
   */
  public void retrievePartition(int partitionId) {
    if (metaPartitionManager.isPartitionOnDisk(partitionId)) {
      ioScheduler.addIOCommand(new LoadPartitionIOCommand(this, partitionId,
          superstep));
      synchronized (partitionAvailable) {
        while (metaPartitionManager.isPartitionOnDisk(partitionId)) {
          try {  
            partitionAvailable.wait();
          } catch (InterruptedException e) {
           //
        }
      }
    }
  }

Continue OutOfCoreIOScheduler.getNextIOCommand

then it will try every type of action, to setup correspond command and return the command.

  command = null;
      for (OutOfCoreOracle.IOAction action : actions) {
        Integer partitionId;
        switch (action) {
        case STORE_MESSAGES_AND_BUFFERS:
          partitionId = oocEngine.getMetaPartitionManager()
              .getOffloadPartitionBufferId(threadId);
          if (partitionId != null) {
            command = new StoreDataBufferIOCommand(oocEngine, partitionId,
                StoreDataBufferIOCommand.DataBufferType.PARTITION);
          } else {
            partitionId = oocEngine.getMetaPartitionManager()
                .getOffloadMessageBufferId(threadId);
            if (partitionId != null) {
              command = new StoreDataBufferIOCommand(oocEngine, partitionId,
                  StoreDataBufferIOCommand.DataBufferType.MESSAGE);
            } else {
              partitionId = oocEngine.getMetaPartitionManager()
                  .getOffloadMessageId(threadId);
              if (partitionId != null) {
                command = new StoreIncomingMessageIOCommand(oocEngine,
                    partitionId);
              }
            }
          }
          break;
        case STORE_PROCESSED_PARTITION:
          partitionId = oocEngine.getMetaPartitionManager()
              .getOffloadPartitionId(threadId);
          if (partitionId != null &&
              oocEngine.getMetaPartitionManager()
                  .isPartitionProcessed(partitionId)) {
            command = new StorePartitionIOCommand(oocEngine, partitionId);
          }
          break;
        case STORE_PARTITION:
          partitionId = oocEngine.getMetaPartitionManager()
              .getOffloadPartitionId(threadId);
          if (partitionId != null) {
            command = new StorePartitionIOCommand(oocEngine, partitionId);
          }
          break;
        case LOAD_UNPROCESSED_PARTITION:
          partitionId = oocEngine.getMetaPartitionManager()
              .getLoadPartitionId(threadId);
          if (partitionId != null &&
              !oocEngine.getMetaPartitionManager()
                  .isPartitionProcessed(partitionId)) {
            command = new LoadPartitionIOCommand(oocEngine, partitionId,
                oocEngine.getSuperstep());
          }
          break;
        case LOAD_TO_SWAP_PARTITION:
          partitionId = oocEngine.getMetaPartitionManager()
              .getLoadPartitionId(threadId);
          if (partitionId != null &&
              !oocEngine.getMetaPartitionManager()
                  .isPartitionProcessed(partitionId) &&
              oocEngine.getMetaPartitionManager().hasProcessedOnMemory()) {
            command = new LoadPartitionIOCommand(oocEngine, partitionId,
                oocEngine.getSuperstep());
          }
          break;
        case LOAD_PARTITION:
          partitionId = oocEngine.getMetaPartitionManager()
              .getLoadPartitionId(threadId);
          if (partitionId != null) {
            if (oocEngine.getMetaPartitionManager()
                .isPartitionProcessed(partitionId)) {
              command = new LoadPartitionIOCommand(oocEngine, partitionId,
                  oocEngine.getSuperstep() + 1);
            } else {
              command = new LoadPartitionIOCommand(oocEngine, partitionId,
                  oocEngine.getSuperstep());
            }
          }
          break;
        case URGENT_LOAD_PARTITION:
          // Do nothing
          break;
        default:
          throw new IllegalStateException("getNextIOCommand: the IO action " +
              "is not defined!");
        }
        if (command != null) {
          break;
        }
      }
      if (command == null) {
        command = new WaitIOCommand(oocEngine, waitInterval);
      }
    } while (!oocEngine.getOracle().approve(command));
    return command;

For example, FixedPartitionsOracle.getNextIOActions.

 if (numPartitions > maxPartitionsInMemory) {
    
      return new IOAction[]{
        IOAction.STORE_PARTITION,
        IOAction.STORE_MESSAGES_AND_BUFFERS};
  }

numPartitons is determined by numPartitionsInMemory. int numPartitionsInMemory = oocEngine.getMetaPartitionManager().getNumInMemoryPartitions();

MetaPartitionManager.addPartition

MetaPartitionManager.numInMemoryPartitions is updated at addPartition and updateCounters.

public void addPartition(int partitionId) {
    MetaPartition meta = new MetaPartition(partitionId);
    MetaPartition temp = partitions.putIfAbsent(partitionId, meta);
    // Check if the given partition is new
    if (temp == null) {
      int index = indexCounter.getAndIncrement();
      checkState(partitionIndex.putIfAbsent(partitionId, index) == null);
      int ownerThread = getOwnerThreadId(partitionId);
      perThreadPartitionDictionary.get(ownerThread).addPartition(meta);
      numInMemoryPartitions.getAndIncrement();
    }