Sparktreaming 例子

最新推荐文章于 2019-01-10 16:06:25 发布

bigdataCoding

最新推荐文章于 2019-01-10 16:06:25 发布

阅读量903

点赞数

分类专栏： Spark 大数据

本文链接：https://blog.csdn.net/UnionIBM/article/details/52902245

版权

大数据同时被 2 个专栏收录

35 篇文章 0 订阅

订阅专栏

Spark

20 篇文章 0 订阅

订阅专栏

def main(args: Array[String]): Unit = {

  val conf = new SparkConf().setMaster("local[*]").setAppName("stremaingtest")
  val sc = new SparkContext(conf)
  val ssc = new StreamingContext(sc, Seconds(10))
  ssc.checkpoint("/tmp/streamTest")
  val sDstream = ssc.socketTextStream("localhost", 8888)
  val words = sDstream.flatMap(_.split(" ")).map(x => (x, 1))
  words.reduceByKeyAndWindow((x: Int, y: Int) => (x + y), Seconds(10), Seconds(10))

  //获取状态
  val statespc = StateSpec.function(transStateSpec _)

 // words.mapWithState 1.6.2以后的API，全局的状态

  val wstate = words.mapWithState(statespc)

  wstate.stateSnapshots().print()

  ssc.start()
  ssc.awaitTermination()

}


def transStateSpec(batchtime: Time, key: String, value: Option[Int], state: State[Long]): Option[(String, Long)] = {
  val sum = value.getOrElse(0).toLong + state.getOption.getOrElse(0L)
  val output = (key, sum)
  state.update(sum)
  Some(output)

}

单利的情况

object SQLContextSingleton {
  @transient  private var instance: SQLContext = _
  def getInstance(sparkContext: SparkContext): SQLContext = {
    if (instance == null) {
      instance = new SQLContext(sparkContext)
    }
    instance
  }
}