spark-streaming中DS的创建方法（用于测试）

由Socket方式构建

package com.zhengkw.stu.day01

import org.apache.spark.streaming.dstream.{DStream, ReceiverInputDStream}
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.{SparkConf, SparkContext}

/**
 * @ClassName:SoketWordCount
 * @author: zhengkw
 * @description:
 * @date: 20/05/16上午 10:40
 * @version:1.0
 * @since: jdk 1.8 scala 2.11.8
 */
object SoketWordCount {
  def main(args: Array[String]): Unit = {
    val conf: SparkConf = new SparkConf().setAppName("SoketWordCount").setMaster("local[2]")
    //获取上下文
    val ssc: StreamingContext = new StreamingContext(conf, Seconds(3))

    val sourceStream: ReceiverInputDStream[String] = ssc.socketTextStream("hadoop102", 9999)
    // 2. 从数据源读取数据, 得到 DStream  (RDD, DataSet, DataFrame)
    val resultStream: DStream[(String, Int)] = sourceStream.flatMap(_.split(" ")).map((_, 1)).reduceByKey(_ + _)
    //打印前N个
    resultStream.print(100)
    // 4. 启动流
    ssc.start()
    // 5. 阻止程序退出
    ssc.awaitTermination()

  }
}

由RDD[T]队列构建

package com.zhengkw.stu.day01

import org.apache.spark.SparkConf
import org.apache.spark.rdd.RDD
import org.apache.spark.streaming.dstream.{DStream, InputDStream}
import org.apache.spark.streaming.{Seconds, StreamingContext}

import scala.collection.mutable

/**
 * @ClassName:RDDQueue
 * @author: zhengkw
 * @description: rdd队列得到DS
 * @date: 20/05/16上午 11:36
 * @version:1.0
 * @since: jdk 1.8 scala 2.11.8
 */
object RDDQueue {
  def main(args: Array[String]): Unit = {
    val conf: SparkConf = new SparkConf().setMaster("local[2]").setAppName("RDDQueue")
    val ssc: StreamingContext = new StreamingContext(conf, Seconds(3))
    //创建一个队列 
    val queue: mutable.Queue[RDD[Int]] = mutable.Queue[RDD[Int]]()
    //获取DS
    //oneAtATime Whether only one RDD should be consumed from the queue in every interval 这里即一次取多个！
    val stream: InputDStream[Int] = ssc.queueStream(queue, oneAtATime = false)
    //聚合DS
    val value: DStream[Int] = stream.reduce(_ + _)
    //不传参数打印10条！
    value.print
    ssc.start()

    //循环添加队列
    while (true) {
      println(queue.size)
      val rdd = ssc.sparkContext.parallelize(1 to 100)
      queue.enqueue(rdd)
      // Thread.sleep(2000)
    }
    ssc.awaitTermination()
  }
}