spark sql 和 spark streaming 结合使用

需要一个构建session单例的类（由于session 存在driver 为一个单例所以需要此类）：

object SparkSessionSingleton {
  @transient private var instance: SparkSession = _

  def getInstance(sparkConf: SparkConf): SparkSession = {
    if (instance == null) {
      instance = SparkSession
        .builder
        .config(sparkConf)
        .getOrCreate()
    }
    instance

  }
}

在streaming 中的算子中使用session的时候需要从RDD中拿出sparkConf 并且使用单例类构建session 单例对象；（注意需要引入隐式转换 ）

object SparkStreaming_SparkSQL {
    case class Record(word:String)
 
    def main(args: Array[String]): Unit = {
        val conf = new SparkConf()
            .setAppName("SparkStreaming and SparkSQL")
            .setMaster("local[2]")
        val ssc = new StreamingContext(conf,Seconds(2))
 
        val lines = ssc.socketTextStream("192.168.112.10",9999)
        val words = lines.flatMap(_.split(" "))
 
        words.foreachRDD{(rdd:RDD[String], time:Time)=>
            val spark = SparkSessionSingleton.getInstance(rdd.sparkContext.getConf)
 
            import spark.implicits._
            val wordsDF = rdd.map(w=>Record(w)).toDF()
 
            wordsDF.createOrReplaceTempView("words")
            spark.sql("select count(*) from words group by word").show()
        }
    }
 
}