【Data Algorithms_Recipes for Scaling up with Hadoop and Spark】Chapter 13 k-Nearest Neighbors

：scala版算法实现

package com.bbw5.dataalgorithms.spark

import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
import breeze.linalg.DenseVector
/**
 * This class solves K-Nearest-Nerigbor join operation using Spark API.
 *
 * knn-query.txt:
 * <unique-record-id><;><a-1><,><a-2><,>...<,><a-d>
 * knn-train.txt:
 * <unique-record-id><;><classification-id><;><b-1><,><b-2><,>...<,><b-d>
 *
 * author:baibaiw5
 */
object SparkKNN {
  def main(args: Array[String]) {
    val sparkConf = new SparkConf().setAppName("SparkKNN")
    val sc = new SparkContext(sparkConf)

    val K = sc.broadcast(3)
    val textQuery = sc.textFile("G:/temp/data/knn-query.txt")
    val textTrain = sc.textFile("G:/temp/data/knn-train.txt")

    val knnQuery = textQuery.map { _.split(";") }.map { array =>
      array(0) -> DenseVector(array(1).split(",").map { _.toDouble })
    }

    val knnTrain = textTrain.map { _.split(";") }.map { array =>
      array(0) -> array(1).toInt -> DenseVector(array(2).split(",").map { _.toDouble })
    }

    knnQuery.collect().foreach(println)
    knnTrain.collect().foreach(println)

    val data = knnQuery.cartesian(knnTrain).map {
      case (tup1, tup2) =>
        //(id,(distance,lable))
        tup1._1 -> (tup1._2.dot(tup2._2) -> tup2._1._2)
    }.groupByKey().mapValues { items =>
      //find nearest k points,then vote for label
      items.toArray.sortBy(_._1).take(K.value).map(item => item._2).groupBy(a => a).mapValues {
        _.size
      }.toArray.sortBy(-_._1).apply(0)._1
    }

    data.collect().foreach(println)
  }
}