import org.apache.spark.SparkConf
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.streaming.dstream.{DStream, ReceiverInputDStream}
object jiachadian {
def functionCreateStreamingContext() ={
val conf =new SparkConf().setAppName(“4”).setMaster("local[]")
val ssc = new StreamingContext(conf,Seconds(2))
val file: ReceiverInputDStream[String] = ssc.socketTextStream(“192.168.11.11”,9000)
ssc.checkpoint(".")
val res: DStream[(String, Int)] = file.flatMap(.split(" ")).map((,1))
val updateFunc= (in:Seq[Int], mi:Option[Int]) => {
val sum =in.sum+mi.getOrElse(0)
Some(sum)
}
res.updateStateByKey(updateFunc).print()
ssc
}
def main(args: Array[String]): Unit = {
val ssc: StreamingContext = StreamingContext.getOrCreate(".",functionCreateStreamingContext)
ssc.start()
ssc.awaitTermination()
}
}*
spark streaming驱动器程序容错
最新推荐文章于 2022-06-20 19:42:33 发布