Spark Graphx连通图(仅做自己学习用)
//创建测试数据
val userRDD: RDD[(String, Map[String, String])] = sc
.parallelize {
Seq(
("M001", Map(("imei", "IM001"), ("mac", "MA001"), ("openudid", "OU001"), ("idfa", "ID001"))),
("M002", Map(("mac", "MA002"), ("openudid", "OU001"))),
("M003", Map(("imei", "IM003"), ("androidid", "AN003"), ("idfa", "ID001"))),
("M004", Map(("imei", "IM004"), ("mac", "MA004"), ("openudid", "OU004"))),
("M005", Map(("androidid", "AN005"), ("idfa", "ID005")))
)
}
// userRDD.foreach(println)
val vertex: RDD[(VertexId, String)] = userRDD
.flatMap {
case (mainID, map) => {
var list = new ListBuffer[(VertexId, String)]()
//方便最后去掉mainID的顶点
list += mainID.hashCode.toLong -> s"mainID#${mainID}"
map.foreach {
case (idValue, id) =>
list += id.hashCode.toLong -> s"${idValue}#${id}"
}
//这里必须把可变的变成不可变的List
list.toList
}
}
// vertex.foreach(println)
val edges: RDD[Edge[String]] = userRDD
.flatMap {
case (mainID, map) => {
val list: ListBuffer[Edge[String]] = new ListBuffer[Edge[String]]()
map.foreach {
case (idName, id) => list += Edge(mainID.hashCode.toLong, id.hashCode.toLong, mainID)
}
//这里必须把可变的变成不可变的List
list.toList
}
}
// edges.foreach(println)
//构建图
val graph: Graph[String, String] = Graph(vertex, edges)
//获得连通图
val ccGraph: Graph[VertexId, String] = graph.connectedComponents()
// connectGraph.edges.foreach(println)
// connectGraph.vertices.foreach(println)
ccGraph
.vertices
.join(vertex)
.map { case (vertexId, (ccVertexId, attr)) =>
(ccVertexId, (vertexId, attr))
}
.groupByKey()
.values
.map { iter =>
iter
.filter {
//将中间你的mainID去掉
case (_, attr) => attr.startsWith("mainID")
}
.map(tuple => tuple._2)
.reduce((mainIdAttr1, mainIdAttr2) => s"$mainIdAttr1, $mainIdAttr2")
}
.foreach(println)
Thread.sleep(1000000)
if (sc.isStopped) sc.stop()
![在这里插入图片描述](https://i-blog.csdnimg.cn/blog_migrate/537282a309c4946adb003689b9590ae3.png)