【Spark】（四）Spark Streaming 结合 SparkSql 实例

最新推荐文章于 2024-07-24 22:35:05 发布

屡傻不改

最新推荐文章于 2024-07-24 22:35:05 发布

阅读量270

点赞数 1

分类专栏： Spark Streaming 文章标签： Spark Streaming

本文链接：https://blog.csdn.net/qianchun22/article/details/112195812

版权

Spark Streaming 专栏收录该内容

5 篇文章 0 订阅

订阅专栏

导入 spark-sql 的依赖

<dependency>
  <groupId>org.apache.spark</groupId>
  <artifactId>spark-sql_2.11</artifactId>
  <version>2.4.5</version>
</dependency>

代码示例

package cn.kgc.kb09.Spark
import org.apache.kafka.clients.consumer.{ConsumerConfig, ConsumerRecord}
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.sql.{Row, SQLContext}
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.streaming.dstream.{DStream, InputDStream}
import org.apache.spark.streaming.kafka010.{ConsumerStrategies, KafkaUtils, LocationStrategies}

object SparkWindowDemo7 {
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setMaster("local[2]").setAppName("KafkaDemo")
    val streamingContext = new StreamingContext(conf,Seconds(10))

    //    streamingContext.checkpoint("checkpoint")

    val kafkaParams: Map[String, String] = Map(
      (ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG -> "192.168.247.201:9092"),
      (ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG -> "org.apache.kafka.common.serialization.StringDeserializer"),
      (ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG -> "org.apache.kafka.common.serialization.StringDeserializer"),
      (ConsumerConfig.GROUP_ID_CONFIG, "kafkaGroup7")
    )


    val kafkaStream: InputDStream[ConsumerRecord[String, String]] =
      KafkaUtils.createDirectStream(
        streamingContext,
        LocationStrategies.PreferConsistent,
        ConsumerStrategies.Subscribe(Set("sparkKafkaDemo"), kafkaParams)
      )


    val numStream: DStream[Row] = kafkaStream.transform((rdd, timestamp) => {
      val sqlContext = SQLContextSingleton.getInstance(rdd.sparkContext)
      import sqlContext.implicits._

      val words = rdd.flatMap(_.value().toString.split("\\s+"))
      val tupple2RDD = words.map((_, 1))

      // 将RDD[(String,Int)]转化成DataFrame，生成临时表
      tupple2RDD.toDF("name", "cn")
        .createOrReplaceTempView("tbwordcount")

      // 在临时表上执行sql语句操作
      val frame = sqlContext.sql("select name,count(cn) from tbwordcount group by name")

      // 将DateFrame转换为DateSet
      val dataSet = frame.map(row => {
        val name = row.getAs[String]("name")
        val count = row.getAs[Long]("cn")
        (name, count)
      })

      dataSet.rdd
      frame.rdd
    })

    numStream.print()

    streamingContext.start()
    streamingContext.awaitTermination()
  }
}

object SQLContextSingleton{
  private var instance:SQLContext =_

  def getInstance(sparkContext: SparkContext):SQLContext ={
    synchronized(
      if (instance == null){
        instance = new SQLContext(sparkContext)
      }
    )
    instance
  }
}