:scala版算法实现
package com.bbw5.dataalgorithms.spark
import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
import breeze.linalg.DenseVector
/**
* This class solves K-Nearest-Nerigbor join operation using Spark API.
*
* knn-query.txt:
* <unique-record-id><;><a-1><,><a-2><,>...<,><a-d>
* knn-train.txt:
* <unique-record-id><;><classification-id><;><b-1><,><b-2><,>...<,><b-d>
*
* author:baibaiw5
*/
object SparkKNN {
def main(args: Array[String]) {
val sparkConf = new SparkConf().setAppName("SparkKNN")
val sc = new SparkContext(sparkConf)
val K = sc.broadcast(3)
val textQuery = sc.textFile("G:/temp/data/knn-query.txt")
val textTrain = sc.textFile("G:/temp/data/knn-train.txt")
val knnQuery = textQuery.map { _.split(";") }.map { array =>
array(0) -> DenseVector(array(1).split(",").map { _.toDouble })
}
val knnTrain = textTrain.map { _.split(";") }.map { array =>
array(0) -> array(1).toInt -> DenseVector(array(2).split(",").map { _.toDouble })
}
knnQuery.collect().foreach(println)
knnTrain.collect().foreach(println)
val data = knnQuery.cartesian(knnTrain).map {
case (tup1, tup2) =>
//(id,(distance,lable))
tup1._1 -> (tup1._2.dot(tup2._2) -> tup2._1._2)
}.groupByKey().mapValues { items =>
//find nearest k points,then vote for label
items.toArray.sortBy(_._1).take(K.value).map(item => item._2).groupBy(a => a).mapValues {
_.size
}.toArray.sortBy(-_._1).apply(0)._1
}
data.collect().foreach(println)
}
}