janusgraph 图数据统计mapreduce

最新推荐文章于 2024-12-25 23:42:34 发布

原创最新推荐文章于 2024-12-25 23:42:34 发布 · 448 阅读

0 ·

CC 4.0 BY-SA版权

文章标签：

#janusgraph #mapreduce #graph

janusgraph mapreduce 专栏收录该内容

1 篇文章

订阅专栏

本文介绍了一个使用MapReduce进行图分析的Scala实现，该算法通过遍历图中的顶点并统计每个标签的出现次数，最终汇总生成图分析结果。具体包括Map阶段对顶点标签计数，Reduce阶段对相同标签的计数求和。

MAPREDUCE:

import java.util

import org.apache.commons.configuration.Configuration
import org.apache.tinkerpop.gremlin.process.computer.{KeyValue, MapReduce}
import org.apache.tinkerpop.gremlin.structure.{Graph, Vertex}

import scala.collection.mutable

class GraphAnalysisMapReduce() extends MapReduce[String, Long, String, Long, mutable.HashMap[String, Long]] {
  val GRAPH_ANALYSIS_MEMORY_KEY = "idmatch.graphAnalysisMapReduce.memorykey"
  val DEFAULT_MEMORY_KEY = "graphAnalysis"

  var memorykey = DEFAULT_MEMORY_KEY

  override def doStage(stage: MapReduce.Stage): Boolean = true

  override def map(vertex: Vertex, emitter: MapReduce.MapEmitter[String, Long]): Unit = {
    emitter.emit(vertex.label(), 1L)
  }

  override def storeState(configuration: Configuration): Unit = {
    super.storeState(configuration)
    configuration.addProperty(GRAPH_ANALYSIS_MEMORY_KEY, this.memorykey)
  }


  override def loadState(graph: Graph, configuration: Configuration): Unit = {
    super.loadState(graph, configuration)
    this.memorykey = configuration.getString(GRAPH_ANALYSIS_MEMORY_KEY, DEFAULT_MEMORY_KEY)
  }


  override def combine(key: String, values: util.Iterator[Long], emitter: MapReduce.ReduceEmitter[String, Long]) {
    this.reduce(key, values, emitter)
  }

  override def reduce(key: String, values: util.Iterator[Long], emitter: MapReduce.ReduceEmitter[String, Long]) {
    var sum = 0L
    while (values.hasNext) {
      sum = sum + values.next()
    }
    emitter.emit(key, sum)
  }


  override def generateFinalResult(keyValues: util.Iterator[KeyValue[String, Long]]): mutable.HashMap[String, Long] = {
    val finalResult = new mutable.HashMap[String, Long]
    while (keyValues.hasNext) {
      val pair = keyValues.next()
      finalResult.put(pair.getKey, pair.getValue)
    }
    return finalResult
  }

  override def getMemoryKey: String = {
    this.memorykey
  }
}

调用：

 val graph = GraphFactory.open("janusgraph_olap.properties")
 graph.configuration().setProperty("gremlin.hadoop.graphWriter", "org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson.GraphSONOutputFormat")
 val result = graph.compute(classOf[SparkGraphComputer]).mapReduce(new GraphAnalysisMapReduce()).submit().get()
 println(s"----graphAnalysis: ${result.memory().get("graphAnalysis")}")