educoder中Spark GraphX—预测社交圈子

第1关：计算连通分量

import org.apache.log4j.{Level, Logger}
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.graphx._
object connectComponents{
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setAppName("connectComponents ").setMaster("local[4]")
    val sc = new SparkContext(conf) //屏蔽日志
    Logger.getLogger("org.apache.spark").setLevel(Level.WARN)
    Logger.getLogger("org.eclipse.jetty.server").setLevel(Level.OFF)
    //**************Begin*************************
    //构造VertexRDD
    val myVertices = sc.parallelize((1L to 8L).map((_,"")))
    //构造EdgeRDD
    val myEdges = sc.parallelize(Array(Edge(1L,2L,""),Edge(4L,5L,""),Edge(4L,6L,""),Edge(5L,6L,""),Edge(5L,7L,""),Edge(7L,6L,"")))
    //构造图Graph[VD,ED]
    val myGraph=Graph(myVertices,myEdges)
    //计算连通分量
    val connectedcomponents = myGraph.connectedComponents.vertices.map(_.swap).groupByKey.map(_._2).collect
    println("")
    //输出结果
    connectedcomponents.foreach(println)
    //**************End**************************
  }
}

第2关：社交圈子预测1

import org.apache.log4j.{Level, Logger}
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.graphx._
object predict1_s{
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setAppName("predict1_s").setMaster("local[4]")
    val sc = new SparkContext(conf) //屏蔽日志
    Logger.getLogger("org.apache.spark").setLevel(Level.WARN)
    Logger.getLogger("org.eclipse.jetty.server").setLevel(Level.OFF)
    // 处理egonet文件的每行数据，返回元组形式的边数据
    def get_edges_from_line(line: String): Array[(Long, Long)] = {
      val ary = line.split(":")
      //划分源顶点和目标顶点
      val srcId = ary(0).toInt
      //根据分隔符把字符串切分为一个数组
      val dstIds = ary(1).split(" ")
      val edges = for {
        dstId <- dstIds
        if (dstId != "")
      } yield {
        //将源顶点和目标顶点组合成元组
        (srcId.toLong, dstId.toLong)
      }
      if (edges.size > 0) edges else Array((srcId, srcId))
    }
    //读取239.egonet文件内容
    val egonet_example = sc.textFile("/root/data/egonets/239.egonet")
    //**************Begin*************************
    //根据定义的get_edges_from_line函数构造由边数据组成的Array
    val edges_array = egonet_example.map(x => get_edges_from_line(x)).collect
    //但现在是一个二维数组，需要用flatten函数转化为一维数组
    val edges = edges_array.flatten
    //构造rawEdges:RDD[(VetexId,VertexId)]
    val g_edges = sc.makeRDD(edges)
    //使用fromEdgeTuples构造图
    val g = Graph.fromEdgeTuples(g_edges,1)
    //**************End**************************
    //**************Begin*************************
    //找出图中的连通分量
    val connectedcomponents = g.connectedComponents.vertices.map(_.swap).groupByKey.map(_._2).collect
    println("")
    //输出结果
    connectedcomponents.foreach(println)
    //**************End**************************
  }
}

第3关：社交圈子预测2

import org.apache.log4j.{Level, Logger}
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.graphx._
object predict2_s{
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setAppName("predict2_s").setMaster("local[4]")
    val sc = new SparkContext(conf) //屏蔽日志
    Logger.getLogger("org.apache.spark").setLevel(Level.WARN)
    Logger.getLogger("org.eclipse.jetty.server").setLevel(Level.OFF)
    // 从path/userId.egonet格式的文件路径中解析出用户ID
    def extract(s: String) = {
      val Pattern = """^.*?(\d+).egonet""".r
      val Pattern(num) = s
      num
    }
    // 处理egonet文件的每行数据，返回元组形式的边数据
    def get_edges_from_line(line: String): Array[(Long, Long)] = {
      val ary = line.split(":")
      val srcId = ary(0).toInt
      val dstIds = ary(1).split(" ")
      val edges = for {
        dstId <- dstIds
        if (dstId != "")
      } yield {
        (srcId.toLong, dstId.toLong)
      }
      if (edges.size > 0) edges else Array((srcId, srcId))
    }
    // 根据文件内容构造边元组
    def make_edges(contents: String) = {
      val lines = contents.split("\n")
      // 根据get_edges_from_line构造元组类型的数组，此时unflat为二维数组
      val unflat = for {
        line <- lines
      } yield {
        get_edges_from_line(line)
      }
      //使用flatten函数降维，并返回值
      val flat = unflat.flatten
      flat
    }
    // 构建图对象，执行connectedComponents()操作，返回结果
    def get_circles(flat: Array[(Long, Long)]) = {
      val edges = sc.makeRDD(flat)
      val g = Graph.fromEdgeTuples(edges,1)
      val cc = g.connectedComponents()
      cc.vertices.map(x => (x._2, Array(x._1))).
        reduceByKey( (a,b) => a ++ b).
        values.map(_.mkString(" ")).collect.mkString(";")
    }
    //读取目录内容
    val egonets = sc.wholeTextFiles("/root/data/egonets")
    //**************Begin*************************
    //获取用户ID
    val egonet_numbers = egonets.map(x => extract(x._1)).collect
    //构造边元组
    val egonet_edges = egonets.map(x => make_edges(x._2)).collect
    //根据get_circles函数构造图，并计算连通分量
    val egonet_circles = egonet_edges.toList.map(x => get_circles(x))
    //**************End**************************
    //**************Begin*************************
    //将用户ID与预测的社交圈子组合，需要使用zip函数
    val result = egonet_numbers.zip(egonet_circles).map(x => x._1 + "," + x._2)
    println("")
    println("UserId,Predicted social circles(Every social circle is used ';' separated.)")
    //换行输出最终结果
    println(result.mkString("\n"))
    //**************End**************************
  }
}