微信文章

case class Record(word:String)

object sqlNetWordCount {
  def main(args: Array[String]): Unit = {
    val sparkConf = new SparkConf().setAppName("ForeachRDDApp").setMaster("local[2]")
    val ssc = new StreamingContext(sparkConf, Seconds(5))

    val lines = ssc.socketTextStream("192.168.56.100", 9999)
    val words = lines.flatMap(_.split(" "))
    // Convert RDDs of the words DStream to DataFrame and run SQL query
    words.foreachRDD { (rdd: RDD[String], time: Time) =>
      val spark = SparkSessionSingleton.getInstance(rdd.sparkContext.getConf)
      import spark.implicits._
      // Convert RDD[String] to RDD[case class] to DataFrame
      val wordsDataFrame = rdd.map(w => Record(w)).toDF()

      // Creates a temporary view using the DataFrame
      wordsDataFrame.createOrReplaceTempView("words")

      // Do word count on table using SQL and print it
      val wordCountsDataFrame =
        spark.sql("select word, count(*) as total from words group by word")
      println(s"========= $time =========")
      wordCountsDataFrame.show()
    }
    ssc.start()
    ssc.awaitTermination()
    /**lazily instantiated singleton instance of sparksession**/
    object SparkSessionSingleton{
     @transient private var instance:SparkSession = _
      def getInstance(sparkConf:SparkConf):SparkSession = {
        if(instance == null){
          instance = SparkSession
            .builder()
            .config(sparkConf)
            .getOrCreate()
        }
        instance
      }
    }
  }
}