Spark Graphx连通图（仅做自己学习用）

Spark Graphx连通图（仅做自己学习用）

//创建测试数据
val userRDD: RDD[(String, Map[String, String])] = sc
  .parallelize {
    Seq(
      ("M001", Map(("imei", "IM001"), ("mac", "MA001"), ("openudid", "OU001"), ("idfa", "ID001"))),
      ("M002", Map(("mac", "MA002"), ("openudid", "OU001"))),
      ("M003", Map(("imei", "IM003"), ("androidid", "AN003"), ("idfa", "ID001"))),
      ("M004", Map(("imei", "IM004"), ("mac", "MA004"), ("openudid", "OU004"))),
      ("M005", Map(("androidid", "AN005"), ("idfa", "ID005")))
    )
  }

//    userRDD.foreach(println)

val vertex: RDD[(VertexId, String)] = userRDD
  .flatMap {
    case (mainID, map) => {
      var list = new ListBuffer[(VertexId, String)]()
      //方便最后去掉mainID的顶点
      list += mainID.hashCode.toLong -> s"mainID#${mainID}"
      map.foreach {
        case (idValue, id) =>
          list += id.hashCode.toLong -> s"${idValue}#${id}"
      }
      //这里必须把可变的变成不可变的List
      list.toList
    }
  }

//    vertex.foreach(println)

val edges: RDD[Edge[String]] = userRDD
  .flatMap {
    case (mainID, map) => {
      val list: ListBuffer[Edge[String]] = new ListBuffer[Edge[String]]()
      map.foreach {
        case (idName, id) => list += Edge(mainID.hashCode.toLong, id.hashCode.toLong, mainID)
      }
      //这里必须把可变的变成不可变的List
      list.toList
    }
  }

//    edges.foreach(println)

//构建图
val graph: Graph[String, String] = Graph(vertex, edges)

//获得连通图
val ccGraph: Graph[VertexId, String] = graph.connectedComponents()

//    connectGraph.edges.foreach(println)
//    connectGraph.vertices.foreach(println)

ccGraph
  .vertices
  .join(vertex)
  .map { case (vertexId, (ccVertexId, attr)) =>
    (ccVertexId, (vertexId, attr))
  }
  .groupByKey()
  .values
  .map { iter =>
    iter
      .filter {
        //将中间你的mainID去掉
        case (_, attr) => attr.startsWith("mainID")
      }
      .map(tuple => tuple._2)
      .reduce((mainIdAttr1, mainIdAttr2) => s"$mainIdAttr1, $mainIdAttr2")
  }
  .foreach(println)

Thread.sleep(1000000)

if (sc.isStopped) sc.stop()