《深度实践Spark机器学习》第13章 使用Spark Streaming构建在线学习模型

抓住一只：

https://www.jianshu.com/p/82c95217e429

13.3 Spark Streaming应用实例 172

// 先启动nc，端口为9999
nc  -lk  9999

// 启动spark-shell
spark-shell --num-executors 1 --total-executor-cores 3 --executor-memory 512m

// 导入类或包
import org.apache.spark.SparkConf
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.SparkSession
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.{Seconds, StreamingContext, Time}
import spark.implicits._
 
// 创建一个间隔时间为3秒的context
val ssc = new StreamingContext(sc, Seconds(3))

// 创建一个socket stream ，基于本机9999端口
val lines = ssc.socketTextStream("127.0.0.1", 9999)
val words = lines.flatMap(_.split(" "))

// 为便于使用SQL进行统计，把DStream的RDD转换为DataFrame。
// 把RDD[String] 转换为RDD[case class] ，最后转换为DataFrame
case class Record(word: String)
 
words.foreachRDD { (rdd:RDD[String], time:Time) =>
val wordsDataFrame = rdd.map(w => Record(w)).toDF()
 
// 创建一个临时视图
wordsDataFrame.createOrReplaceTempView("words")
// 使用SQL进行统计
val wordCountsDataFrame = spark.sql("select word, count(*) as total from words group by word")
println(s"========= $time =========")
wordCountsDataFrame.show()
}

ssc.start()
ssc.awaitTermination()

// 在启动了nc的界面输入
ok ok  m  p  py py py

// 在spark shell界面，可以看到如下输出

+----+-----+                                                                    
|word|total|
+----+-----+
|   m|    1|
|  ok|    2|
|   p|    1|
|  py|    3|
|    |    3|
+----+-----+

13.4 Spark Streaming在线学习实例（未运行测试）
// 导入需要的类
import org.apache.spark.mllib.linalg.Vectors
import org.apache.spark.mllib.feature.StandardScaler
import breeze.linalg.DenseVector
import org.apache.spark.mllib.regression.LabeledPoint
import org.apache.spark.mllib.regression.StreamingLinearRegressionWithSGD
import org.apache.spark.streaming._
import org.apache.spark.streaming.StreamingContext._
 
 
// 交互式编程
val ssc = new StreamingContext(sc, Seconds(10))
val stream = ssc.textFileStream("file:///XX")
 
val NumFeatures = 11
val zeroVector = DenseVector.zeros[Double](NumFeatures)
val model = new StreamingLinearRegressionWithSGD()
.setInitialWeights(Vectors.dense(zeroVector.data))
.setNumIterations(20)
.setRegParam(0.8)
.setStepSize(0.01) 
 

// 创建一个含标签的数据流
val labeledStream = stream.map { line =>
val split = line.split(";")
val y = split(11).toDouble
val features=split.slice(0,11).map(_.toDouble)
LabeledPoint(label = y, features = Vectors.dense(features))
}  

// 在数据流上训练测试模型。    
model.trainOn(labeledStream)
model.predictOnValues(labeledStream.map(lp => (lp.label, lp.features))).print()

// 启动Spark Streaming
ssc.start()
ssc.awaitTermination()