set partitons
USER_PARTITION_COUNT.set(conf, 100);
add local cache
Vertex reader read vertex and i’ts edges information, first store in local cache.
catch size: 629145.
if (workerMessageSize >= maxVerticesSizePerWorker) {
sendOut()
}
maxVerticesSizePerWorker: default 512k
giraph.useOutOfCoreGraph is used to use DiskBackedPartitionStore
determine partition count
public static int computePartitionCount(int availableWorkerCount,
ImmutableClassesGiraphConfiguration conf) {
if (availableWorkerCount == 0) {
throw new IllegalArgumentException(
"computePartitionCount: No available workers");
}
int userPartitionCount = USER_PARTITION_COUNT.get(conf);
int partitionCount;
if (userPartitionCount == USER_PARTITION_COUNT.getDefaultValue()) {
float multiplier = GiraphConstants.PARTITION_COUNT_MULTIPLIER.get(conf);
partitionCount = Math.max(
(int) (multiplier * availableWorkerCount * availableWorkerCount), 1);
int minPartitionsPerComputeThread =
MIN_PARTITIONS_PER_COMPUTE_THREAD.get(conf);
int totalComputeThreads =
NUM_COMPUTE_THREADS.get(conf) * availableWorkerCount;
partitionCount = Math.max(partitionCount,
minPartitionsPerComputeThread * totalComputeThreads);
} else {
partitionCount = userPartitionCount;
}
if (LOG.isInfoEnabled()) {
LOG.info("computePartitionCount: Creating " +
partitionCount + " partitions.");
}
return partitionCount;
}
determine partition to worker map
public class HashPartitionerFactory<I extends WritableComparable,
V extends Writable, E extends Writable>
extends GraphPartitionerFactory<I, V, E> {
@Override
public int getWorker(int partition, int partitionCount, int workerCount) {
return partition % workerCount;
}
}