SparkStreaming中使用SparkSQL实现WordCount

SparkStreaming基于spark sql和dataframe实现wordcount

package com.bigdata.wb.spark

import org.apache.spark.SparkConf
import org.apache.spark.sql.SparkSession
import org.apache.spark.streaming.{Seconds, StreamingContext}

/**
  * @ author spencer
  * @ date 2020/7/15 14:43
  *
  */
object SparkWindowSQLDemo {

  def main(args: Array[String]): Unit = {

    val conf = new SparkConf().setAppName("SparkWindowSQLDemo").setMaster("local[*]")

    val ssc = new StreamingContext(conf, Seconds(2))

    val sc = ssc.sparkContext

    ssc.checkpoint("file:///D:\\IdeaProjects\\spark-hbase\\chck")

    val wordDStream = ssc.socketTextStream("localhost", 7777)

    //使用spark sql实现wordCount
    wordDStream.flatMap(_.split(" "))
      .countByValueAndWindow(Seconds(10), Seconds(6))
      .foreachRDD(rdd => {
        // Get the singleton instance of SparkSession
        val spark = SparkSession.builder()
          .config(rdd.sparkContext.getConf)
          .getOrCreate()

      import spark.implicits._
      val wordsDF = rdd.toDF("word")
      wordsDF.createOrReplaceTempView("words")

      spark.sql("select word, count(word) total from words group by word").show()
    })

    ssc.start()
    ssc.awaitTermination()
  }
}