Giraph Partiton Strategy

set partitons

USER_PARTITION_COUNT.set(conf, 100);

add local cache

Vertex reader read vertex and i’ts edges information, first store in local cache.
catch size: 629145.

 if (workerMessageSize >= maxVerticesSizePerWorker) {
    sendOut()
 }
 maxVerticesSizePerWorker: default 512k

giraph.useOutOfCoreGraph is used to use DiskBackedPartitionStore

determine partition count

public static int computePartitionCount(int availableWorkerCount,
      ImmutableClassesGiraphConfiguration conf) {
    if (availableWorkerCount == 0) {
      throw new IllegalArgumentException(
          "computePartitionCount: No available workers");
    }

    int userPartitionCount = USER_PARTITION_COUNT.get(conf);
    int partitionCount;
    if (userPartitionCount == USER_PARTITION_COUNT.getDefaultValue()) {
      float multiplier = GiraphConstants.PARTITION_COUNT_MULTIPLIER.get(conf);
      partitionCount = Math.max(
          (int) (multiplier * availableWorkerCount * availableWorkerCount), 1);
      int minPartitionsPerComputeThread =
          MIN_PARTITIONS_PER_COMPUTE_THREAD.get(conf);
      int totalComputeThreads =
          NUM_COMPUTE_THREADS.get(conf) * availableWorkerCount;
      partitionCount = Math.max(partitionCount,
          minPartitionsPerComputeThread * totalComputeThreads);
    } else {
      partitionCount = userPartitionCount;
    }
    if (LOG.isInfoEnabled()) {
      LOG.info("computePartitionCount: Creating " +
          partitionCount + " partitions.");
    }
    return partitionCount;
  }

determine partition to worker map

public class HashPartitionerFactory<I extends WritableComparable,
    V extends Writable, E extends Writable>
    extends GraphPartitionerFactory<I, V, E> {


  @Override
  public int getWorker(int partition, int partitionCount, int workerCount) {
    return partition % workerCount;
  }
}