大数据系列-SPARK-STREAMING流数据queue
package com.test
import org.apache.spark.SparkConf
import org.apache.spark.rdd.RDD
import org.apache.spark.streaming.{Seconds, StreamingContext}
import scala.collection.mutable
object SparkStreamingQueue {
def main(args: Array[String]): Unit = {
val sparkConf = new SparkConf().setAppName("SparkStreamingQueue").setMaster("local[*]")
val streamingContext = new StreamingContext(sparkConf, Seconds(5))
val rddQueue = new mutable.Queue[RDD[Int]]()
streamingContext.queueStream(rddQueue, oneAtATime = false)
.map((_, 1))
.reduceByKey(_ + _)
.print()
streamingContext.start()
for (k <- 0 to 5000) {
rddQueue += streamingContext.sparkContext.makeRDD(1 to 300, 10)
Thread.sleep(2000)
}
streamingContext.awaitTermination()
}
}