spark实现共同好友

基础数据：

用户,好友1 好友2 好友3 .....

A1,A2 A3 A4 A5 A6
A2,A1 A3 A4 A6
A3,A1 A2 A4 A5 A6
A4,A1 A2 A3
A5,A1 A3 A7
A6,A1 A4
A7,A2 A5
代码实现：

import org.apache.spark.{SparkConf, SparkContext}
object ComFriendV2 {

  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setAppName("common_friends").setMaster("local")
    val sc = new SparkContext(conf)
    val rdd = sc.textFile("e://friends.txt");
    rdd.flatMap(line => {
      val data = line.split(",");
      val person = data(0);
      val friends = data(1)
      val friendArray = friends.split(" ");
      for (item <- friendArray)
        yield (item, person)
    }).combineByKey(
      List[String](_),
      (a: List[String], b: String) => a :+ b,
      (x: List[String], y: List[String]) => x ++ y)
      .flatMap(pair => {
        val friend = pair._1
        val persons = pair._2.toArray
        var map = Map[String, String]()
        for (i <- 0 until persons.length) {
          for (j <- (i + 1) until persons.length) {
            val personI = persons(i)
            val personJ = persons(j)
            if (personI > personJ) {
              map += ((personI + "-" + personJ) -> friend)
            } else {
              map += ((personJ + "-" + personI) -> friend)
            }
          }
        }
        map
      }).reduceByKey(_ + "," + _).foreach(println)
  }
}

结果值：

(A4-A1,A3,A2)
(A6-A2,A4,A1)
(A7-A1,A5,A2)
(A5-A1,A3)
(A2-A1,A3,A6,A4)
(A7-A3,A5,A2)
(A7-A4,A2)
(A4-A3,A1,A2)
(A5-A2,A3,A1)
(A4-A2,A3,A1)
(A6-A5,A1)
(A6-A4,A1)
(A6-A1,A4)
(A3-A1,A6,A4,A5,A2)
(A6-A3,A4,A1)
(A3-A2,A6,A4,A1)
(A5-A3,A1)
(A5-A4,A3,A1)

性能优化点后续更新

转载于:https://my.oschina.net/u/2443971/blog/3092978