Vector
vectors.txt
1 2.3 4.5
3 3.1 5.6
4 3.2 7.8
处理vectors.txt文件RDD[String]->RDD[Vector]
package com.yasuofenglei
import org.apache.spark.mllib.linalg.Vectors
import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
object Driver01 {
def main(args: Array[String]): Unit = {
//创建向量类型,传入的类型是Double类型,如果是Int类型,会自动转换
val v1=Vectors.dense(1,2,3.1,4,5)
//
val v2=Vectors.dense(Array[Double](1,2,3,4))
val conf=new SparkConf()
conf.setMaster("local").setAppName("wordcount")
val sc=new SparkContext(conf)
val data=sc.textFile("d://data/ml/vectors.txt",2)
//RDD[String]->RDD[Array[String]]->RDD[Array[Double]]
val r1=data.map{_.split(" ").map{num => num.toDouble}}
.map{arr =>Vectors.dense(arr)}
r1.foreach { println}
}
}
向量标签
package com.yasuofenglei
import org.apache.spark.mllib.linalg.Vectors
import org.apache.spark.mllib.regression.LabeledPoint
import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
object Driver02 {
def main(args: Array[String]): Unit = {
val v1=Vectors.dense(2.2,3.1,4.5)
//创建一个向量标签,1标签值
val lb1 = LabeledPoint(1, v1)
println(lb1)
println(lb1.features)//获取向量标签中的向量
println(lb1.label)//获取标签值
val conf=new SparkConf()
conf.setMaster("local").setAppName("wordcount")
val sc=new SparkContext(conf)
val data=sc.textFile("d://data/ml/vectors.txt",2)
val r1=data.map{line=>
val info=line.split(" ")
val label=info.head.toDouble
val features=info.drop(1).map{
num => num.toDouble
}
LabeledPoint(label,Vectors.dense(features))
}
r1.foreach{println}
}
}
Spark计算工具类
package com.yasuofenglei
import org.apache.spark._
import org.apache.spark.mllib.linalg._
import org.apache.spark.mllib.stat.Statistics
/**
* Spark计算工具类
*/
object Driver03 {
def main(args: Array[String]): Unit = {
var conf=new SparkConf().setMaster("local").setAppName("statistic")
var sc=new SparkContext(conf)
val r1=sc.makeRDD(List(1,2,3,4,5))
//RDD[Int]->RDD[Vector]
val r2=r1.map{ num => Vectors.dense(num)}
val result=Statistics.colStats(r2)
println(result.max)
println(result.min)
println(result.mean)//均值
println(result.variance)//方差
println(result.count)
println(result.numNonzeros)//不为0的元素个数
println(result.normL1)//计算曼哈顿距离
println(result.normL2)//计算欧式距离
}
}
欧氏距离
欧氏距离(Euclidean Distance)
在二维和三维空间中的欧氏距离的就是两点之间的距离。
二维空间的欧氏距离: