Spark 机器学习之Kmeans算法实践

数据格式说明

id为出租车司机id,tid,lat为经纬度,time是时间(HHmmss)

目标

通过Kmean探索不同地区接客最多的时间

实践代码

import org.apache.spark.ml.clustering.KMeans
import org.apache.spark.ml.feature.VectorAssembler
import org.apache.spark.ml.linalg
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.types.{StructField, StructType}
import org.apache.spark.sql.types._
import org.apache.spark.sql.functions._

import scala.collection.mutable
object te4 {
  def main(arg:Array[String]):Unit={
    val spark = SparkSession.builder().appName("Taxi1").master("local[*]").getOrCreate()
    //toDo 为读取的数据创建schema
    
    val taxiSchema = StructType(Array(
      StructField("id",IntegerType,true),
      StructField("tid",DoubleType,true),
      StructField("lat",DoubleType,true),
      StructField("time",StringType,true)
    ))
    val path = "F:\\data\\taxi.csv"
    val data = spark.read.schema(taxiSchema).csv(path)
    //toDo 将tid和lat转换成训练使用的Vector类型
    val assembler = new VectorAssembler()
    val tid_lat = Array("tid","lat")
    val frame = assembler.setInputCols(tid_lat).setOutputCol("feature").transform(data)
    //toDo 按照8：2的比例随即分割数据，分别用于训练和测试
    val splitRait = Array(0.8, 0.2)
    val Array(train, test) = frame.randomSplit(splitRait)



    //toDo 建立Kmeans，设置类别数为10
    val kmeans = new KMeans().setK(10).setFeaturesCol("feature").setPredictionCol("prediction")
    //toDo 模型训练
    val model = kmeans.fit(train)

    import spark.implicits._
   // val structType: StructType = StructType(Array(StructField("prediction",IntegerType)))
    val tuples = new mutable.ArrayBuffer[Tuple2[Int,linalg.Vector]]()
    var index=0
    for(i<- model.clusterCenters){
      tuples.append((index,i))
      index+=1
    }
    tuples.toDF("id","center").createOrReplaceTempView("test2")
    spark.sql("select * from test2").show()
    //toDo 使用模型预测 测试数据
    val testResult = model.transform(test)
    testResult.show(false)
    
    testResult.createOrReplaceTempView("test")
    spark.sql("select * from(select *,row_number() over(partition by prediction order by cou desc) as paiming from (select prediction,collect_set(id),substring(time,0,2) as hour,count(*) as cou from test group by prediction,substring(time,0,2) order by cou desc)) t1  left join test2 t2 on t1.prediction=t2.id where t1.paiming=1 ").show()
  }
}