大数据之SparkDStream 创建 完整使用 (第十三章)

一、RDD 队列

1、用法及说明

测试过程中，可以通过使用 ssc.queueStream(queueOfRDDs)来创建 DStream，每一个推送到这个队列中的 RDD，都会作为一个 DStream 处理。

2、案例实操

➢ 需求：循环创建几个 RDD，将 RDD 放入队列。通过 SparkStream 创建 Dstream，计算WordCount

1) 编写代码

package com.spack.bigdata.streaming

import org.apache.spark.SparkConf
import org.apache.spark.rdd.RDD
import org.apache.spark.streaming.dstream.{DStream, ReceiverInputDStream}
import org.apache.spark.streaming.{Seconds, StreamingContext}

import scala.collection.mutable

/**
 * DStream 创建
 * RDD 队列
 */
object SparkStreaming02_Queue {
  def main(args: Array[String]): Unit = {

    //创建环境对象
    //StreamingContext 创建时、需要传递两个参数
    //第一个参数表示环境变量
    val sparkConf = new SparkConf().setMaster("local[*]").setAppName("SparkStreaming")

    //第二个参数表示批量处理的周期（采集周期）---Seconds以秒为单位
    val scc = new StreamingContext(sparkConf, Seconds(3))

    //创建RDD队列
    val RDDQueue = new mutable.Queue[RDD[Int]]()

    val inputStream = scc.queueStream(RDDQueue, oneAtATime = false)
    val mappedStream = inputStream.map((_, 1))
    val reducedStream = mappedStream.reduceByKey(_ + _)

    //TODO 打印结果
    reducedStream.print()

    //1、启动采集器
    scc.start()

    for (i <- 1 to 5) {
      RDDQueue += scc.sparkContext.makeRDD(1 to 300, 10)
      Thread.sleep(2000)
    }
    //2、等待采集器的关闭
    scc.awaitTermination()
  }


}

结果展示：

Time: 1658115024000 ms
-------------------------------------------
(224,1)
(160,1)
(296,1)
(96,1)
(56,1)
(112,1)
(120,1)
(280,1)
(16,1)
(184,1)
...

-------------------------------------------
Time: 1658115027000 ms
-------------------------------------------
(224,1)
(160,1)
(296,1)
(96,1)
(56,1)
(112,1)
(120,1)
(280,1)
(16,1)
(184,1)
...

-------------------------------------------
Time: 1658115030000 ms
-------------------------------------------
(224,2)
(160,2)
(296,2)
(96,2)
(56,2)
(112,2)
(120,2)
(280,2)
(16,2)
(184,2)
...

-------------------------------------------
Time: 1658115033000 ms
-------------------------------------------
(224,1)
(160,1)
(296,1)
(96,1)
(56,1)
(112,1)
(120,1)
(280,1)
(16,1)
(184,1)
...

二、自定义数据源

1、用法及说明

需要继承 Receiver，并实现 onStart、onStop 方法来自定义数据源采集。

2、案例实操

需求：自定义数据源，实现监控某个端口号，获取该端口号内容。

1) 自定义数据源

package com.spack.bigdata.streaming

import org.apache.spark.SparkConf
import org.apache.spark.rdd.RDD
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.dstream.ReceiverInputDStream
import org.apache.spark.streaming.receiver.Receiver
import org.apache.spark.streaming.{Seconds, StreamingContext}

import scala.collection.mutable
import scala.util.Random

/**
 * DStream 创建
 * 自定义数据采集器
 */
object SparkStreaming03_DIY {
  def main(args: Array[String]): Unit = {

    //创建环境对象
    //StreamingContext 创建时、需要传递两个参数
    //第一个参数表示环境变量
    val sparkConf = new SparkConf().setMaster("local[*]").setAppName("SparkStreaming")

    //第二个参数表示批量处理的周期（采集周期）---Seconds以秒为单位
    val scc = new StreamingContext(sparkConf, Seconds(3))

    //调用自定义数据采集器
    val messageDS: ReceiverInputDStream[String] = scc.receiverStream(new MyReceiver)
    messageDS.print()

    //TODO 打印结果


    //1、启动采集器
    scc.start()

    //2、等待采集器的关闭
    scc.awaitTermination()
  }


  /**
   * 自定义数据采集器
   * 1、继承Receiver,定义泛型，传递参数
   * 2、重写方法
   */
  class MyReceiver extends Receiver[String](StorageLevel.MEMORY_ONLY) {

    private var flg = true;

    override def onStart(): Unit = {
      new Thread(new Runnable {
        override def run(): Unit = {
          while (flg) {
            val message = "采集的数据为：" + new Random().nextInt(10).toString
            store(message) //封装数据
            Thread.sleep(500)
          }
        }
      }).start()

    }

    override def onStop(): Unit = {
      flg = false
    }
  }
}

三、 Kafka 数据源（面试、开发重点）

1、版本选型

ReceiverAPI：需要一个专门的 Executor 去接收数据，然后发送给其他的 Executor 做计算。存在
的问题，接收数据的 Executor 和计算的 Executor 速度会有所不同，特别在接收数据的 Executor
速度大于计算的 Executor 速度，会导致计算数据的节点内存溢出。早期版本中提供此方式，当 前版本不适用

DirectAPI：是由计算的 Executor 来主动消费 Kafka 的数据，速度由自身控制。

2、Kafka 0-8 Receiver 模式（当前版本不适用）

1） 需求：通过 SparkStreaming 从 Kafka 读取数据，并将读取过来的数据做简单计算，最终打印
到控制台。

2）导入依赖

<dependency>
 <groupId>org.apache.spark</groupId>
 <artifactId>spark-streaming-kafka-0-8_2.11</artifactId>
 <version>2.4.5</version>
</dependency>

3）编写代码

package com.atguigu.kafka
import org.apache.spark.SparkConf
import org.apache.spark.streaming.dstream.ReceiverInputDStream
import org.apache.spark.streaming.kafka.KafkaUtils
import org.apache.spark.streaming.{Seconds, StreamingContext}
object ReceiverAPI {
 def main(args: Array[String]): Unit = {
 //1.创建 SparkConf
 val sparkConf: SparkConf = new 
SparkConf().setAppName("ReceiverWordCount").setMaster("local[*]")
 //2.创建 StreamingContext
 val ssc = new StreamingContext(sparkConf, Seconds(3))
 //3.读取 Kafka 数据创建 DStream(基于 Receive 方式)
 val kafkaDStream: ReceiverInputDStream[(String, String)] = 
KafkaUtils.createStream(ssc,
 "linux1:2181,linux2:2181,linux3:2181",
 "atguigu",
 Map[String, Int]("atguigu" -> 1))
 //4.计算 WordCount
 kafkaDStream.map { case (_, value) =>
 (value, 1)
 }.reduceByKey(_ + _)
 .print()
 //5.开启任务
 ssc.start()
 ssc.awaitTermination()
 } }

3、Kafka 0-8 Direct 模式（当前版本不适用）

1）需求：通过 SparkStreaming 从 Kafka 读取数据，并将读取过来的数据做简单计算，最终打印
到控制台。

2）导入依赖

<dependency>
 <groupId>org.apache.spark</groupId>
 <artifactId>spark-streaming-kafka-0-8_2.11</artifactId>
 <version>2.4.5</version>
</dependency>

3）编写代码（自动维护 offset）

import kafka.serializer.StringDecoder
import org.apache.kafka.clients.consumer.ConsumerConfig
import org.apache.spark.SparkConf
import org.apache.spark.streaming.dstream.InputDStream
import org.apache.spark.streaming.kafka.KafkaUtils
import org.apache.spark.streaming.{Seconds, StreamingContext}
object DirectAPIAuto02 {
 val getSSC1: () => StreamingContext = () => {
 val sparkConf: SparkConf = new 
SparkConf().setAppName("ReceiverWordCount").setMaster("local[*]")
 val ssc = new StreamingContext(sparkConf, Seconds(3))
 ssc
 }
 def getSSC: StreamingContext = {
 //1.创建 SparkConf
 val sparkConf: SparkConf = new 
SparkConf().setAppName("ReceiverWordCount").setMaster("local[*]")
 //2.创建 StreamingContext
 val ssc = new StreamingContext(sparkConf, Seconds(3))
 //设置 CK
 ssc.checkpoint("./ck2")
 //3.定义 Kafka 参数
 val kafkaPara: Map[String, String] = Map[String, String](
 ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG -> 
"linux1:9092,linux2:9092,linux3:9092",
 ConsumerConfig.GROUP_ID_CONFIG -> "atguigu"
 )
 //4.读取 Kafka 数据
 val kafkaDStream: InputDStream[(String, String)] = 
KafkaUtils.createDirectStream[String, String, StringDecoder, StringDecoder](ssc,
 kafkaPara,
 Set("atguigu"))
 //5.计算 WordCount
 kafkaDStream.map(_._2)
 .flatMap(_.split(" "))
 .map((_, 1))
 .reduceByKey(_ + _)
 .print()
 //6.返回数据
 ssc
 }
 def main(args: Array[String]): Unit = {
 //获取 SSC
 val ssc: StreamingContext = StreamingContext.getActiveOrCreate("./ck2", () => 
getSSC)
 //开启任务
 ssc.start()
 ssc.awaitTermination()
 } }

4）编写代码（手动维护 offset）

import kafka.common.TopicAndPartition
import kafka.message.MessageAndMetadata
import kafka.serializer.StringDecoder
import org.apache.kafka.clients.consumer.ConsumerConfig
import org.apache.spark.SparkConf
import org.apache.spark.streaming.dstream.{DStream, InputDStream}
import org.apache.spark.streaming.kafka.{HasOffsetRanges, KafkaUtils,
OffsetRange}
import org.apache.spark.streaming.{Seconds, StreamingContext}
object DirectAPIHandler {
 def main(args: Array[String]): Unit = {
 //1.创建 SparkConf
 val sparkConf: SparkConf = new 
SparkConf().setAppName("ReceiverWordCount").setMaster("local[*]")
 //2.创建 StreamingContext
 val ssc = new StreamingContext(sparkConf, Seconds(3))
 //3.Kafka 参数
 val kafkaPara: Map[String, String] = Map[String, String](
 ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG -> 
"hadoop102:9092,hadoop103:9092,hadoop104:9092",
 ConsumerConfig.GROUP_ID_CONFIG -> "atguigu"
 )
 //4.获取上一次启动最后保留的 Offset=>getOffset(MySQL)
 val fromOffsets: Map[TopicAndPartition, Long] = Map[TopicAndPartition, 
Long](TopicAndPartition("atguigu", 0) -> 20)
 //5.读取 Kafka 数据创建 DStream
 val kafkaDStream: InputDStream[String] = KafkaUtils.createDirectStream[String, 
String, StringDecoder, StringDecoder, String](ssc,
 kafkaPara,
 fromOffsets,
 (m: MessageAndMetadata[String, String]) => m.message())
 //6.创建一个数组用于存放当前消费数据的 offset 信息
 var offsetRanges = Array.empty[OffsetRange]
 //7.获取当前消费数据的 offset 信息
 val wordToCountDStream: DStream[(String, Int)] = kafkaDStream.transform { rdd 
=>
 offsetRanges = rdd.asInstanceOf[HasOffsetRanges].offsetRanges
 rdd
 }.flatMap(_.split(" "))
 .map((_, 1))
 .reduceByKey(_ + _)
 //8.打印 Offset 信息
 wordToCountDStream.foreachRDD(rdd => {
 for (o <- offsetRanges) {
 println(s"${o.topic}:${o.partition}:${o.fromOffset}:${o.untilOffset}")
 }
 rdd.foreach(println)
 })
 //9.开启任务
 ssc.start()
 ssc.awaitTermination()
 } }

4、 Kafka 0-10 Direct 模式（适用）

1）需求：通过 SparkStreaming 从 Kafka 读取数据，并将读取过来的数据做简单计算，最终打印
到控制台。

2）导入依赖

<dependency>
 <groupId>org.apache.spark</groupId>
 <artifactId>spark-streaming-kafka-0-10_2.12</artifactId>
 <version>3.0.0</version>
</dependency>
<dependency>
 <groupId>com.fasterxml.jackson.core</groupId>
 <artifactId>jackson-core</artifactId>
 <version>2.10.1</version>
 </dependency>

3）编写代码

package com.spack.bigdata.streaming

import org.apache.kafka.clients.consumer.{ConsumerConfig, ConsumerRecord}
import org.apache.spark.SparkConf
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.streaming.dstream.{InputDStream, ReceiverInputDStream}
import org.apache.spark.streaming.kafka010.{ConsumerStrategies, KafkaUtils, LocationStrategies}

/**
 * SparkStreaming
 * DStream创建
 * Kafka数据源
 */
object SparkStreaming04_Kafka {
  def main(args: Array[String]): Unit = {
    //创建环境对象
    //StreamingContext 创建时、需要传递两个参数
    //第一个参数表示环境变量
    val sparkConf = new SparkConf().setMaster("local[*]").setAppName("SparkStreaming")

    //第二个参数表示批量处理的周期（采集周期）---Seconds以秒为单位
    val scc = new StreamingContext(sparkConf, Seconds(3))

    //设置Kafka参数
    val kaConfin = setKafKaConfin

    //读取KafKa数据创建DStream
    val KafkaDataDS: InputDStream[ConsumerRecord[String, String]] = KafkaUtils.createDirectStream[String, String](
      scc, //上下文的环境对象
      LocationStrategies.PreferConsistent, //位置的策略、采集节点、和计算该如何做匹配、TODO 自动选择、由框架来匹配
      ConsumerStrategies.Subscribe[String, String](Set("atguiguNew"), kaConfin) //消费者策略
    )

    KafkaDataDS.map(_.value()).print()
    
    //开启后台启动
    scc.start()
    scc.awaitTermination()
  }


  /**
   * Kafka配置
   * @return
   */
  def setKafKaConfin(): Map[String, Object] = {
    //3.定义 Kafka 参数
    val kafkaPara: Map[String, Object] = Map[String, Object](
      ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG -> "hadoop102:9092,hadoop103:9092,hadoop104:9092",
      ConsumerConfig.GROUP_ID_CONFIG -> "atguigu",
      "key.deserializer" -> "org.apache.kafka.common.serialization.StringDeserializer",
      "value.deserializer" -> "org.apache.kafka.common.serialization.StringDeserializer"
    )
    kafkaPara
  }
}

5、生产消息

1）发送

bin/kafka-console-producer.sh --broker-list hadoop102:9092 --topic atguiguNew

2）查询

bin/kafka-topics.sh --zookeeper hadoop102:2181 --list

3）创建

bin/kafka-topics.sh --zookeeper hadoop102:2181 --create --topic atguiguNew --partitions 3 --replication-factor 2