package org.lm.spark.streaming import org.apache.spark.streaming.{Seconds, StreamingContext} import org.apache.spark.{SparkConf, SparkContext} object StatefulWordCountOnLine { def main(args: Array[String]): Unit = {
//定义updateByKey使用到的函数 val updateFunc=(values:Seq[Int],state:Option[Int])=>{ val currentCount=values.foldLeft(0)(_+_) val previousCount=state.getOrElse(0) Some(currentCount+previousCount) } val conf=new SparkConf().setAppName("Stateful Word Count On Line").setMaster("spark://192.168.189.128:7077") val sc=new SparkContext(conf) val ssc=new StreamingContext(sc,Seconds(5))
//定义持久化存储位置,很重要,必须定义 ssc.checkpoint("hdfs://192.168.189.128:9000/user/input/StatefulWordCount/log") val lines=ssc.socketTextStream("192.168.189.129",9999) val words=lines.flatMap(_.split(" ")) val pairs=words.map(word=>(word,1))
//调用updateStateByKey方法,参数传入前面定义的updateFunc函数 val statefulDStream=pairs.updateStateByKey[Int](updateFunc) statefulDStream.print() ssc.start() ssc.awaitTermination() } }