Spark Streaming和Spark SQL

Spark Streaming中的逻辑除了可以用RDD写，还可以使用Spark SQL来写。

需求：实时读取kafka数据，使用Spark SQL实现wordcount

import org.apache.kafka.clients.consumer.{ConsumerConfig, ConsumerRecord}
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.sql.{DataFrame, Row, SQLContext}
import org.apache.spark.streaming.dstream.{DStream, InputDStream}
import org.apache.spark.streaming.kafka010.{ConsumerStrategies, KafkaUtils, LocationStrategies}
import org.apache.spark.streaming.{Seconds, StreamingContext}

object SparkWindowDemo {
  def main(args: Array[String]): Unit = {
    val conf: SparkConf = new SparkConf().setAppName("SparkWindowDemo").setMaster("local[*]")

    val streamingContext = new StreamingContext(conf,Seconds(2))   //批处理时间设置为2秒,也就是采集时间

    streamingContext.checkpoint("checkpoint")


    val kafkaParams: Map[String, String] = Map(
      (ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG -> "192.168.136.20:9092"),
      (ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG -> "org.apache.kafka.common.serialization.StringDeserializer"),
      (ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG -> "org.apache.kafka.common.serialization.StringDeserializer"),
      (ConsumerConfig.GROUP_ID_CONFIG -> "kafkaGroup2")
    )

    val kafkaStream: InputDStream[ConsumerRecord[String, String]] = KafkaUtils.createDirectStream(
      streamingContext,
      LocationStrategies.PreferConsistent,
      ConsumerStrategies.Subscribe(Set("SparkKafkaDemo"), kafkaParams) //SparkKafkaDemo是topic
    )

      val numStream: DStream[Row] = kafkaStream.transform(rdd => {
      val sqlContext: SQLContext = SQLContextSingleton.getInstance(rdd.sparkContext)
      import sqlContext.implicits._
      val words: RDD[String] = rdd.flatMap(_.value().toString.split("\\s+"))
      val tupple2RDD: RDD[(String, Int)] = words.map((_, 1))
      tupple2RDD.toDF("name", "num")
        .createOrReplaceTempView("tbwordcount")

      val frame: DataFrame = sqlContext.sql("select name,count(num) from tbwordcount group by name")
      frame.rdd
    })

    numStream.print()

    streamingContext.start()
    streamingContext.awaitTermination()
  }
}

object SQLContextSingleton{
  @transient private var instance:SQLContext=_
  def getInstance(sparkContext:SparkContext):SQLContext={
    synchronized(
      if(instance==null){
        instance=new SQLContext(sparkContext)
      }
    )
    instance
  }
}

此项目的pom依赖如下：

    <dependency>
      <groupId>org.apache.kafka</groupId>
      <artifactId>kafka_2.11</artifactId>
      <version>2.0.0</version>
    </dependency>

    <dependency>
      <groupId>org.apache.kafka</groupId>
      <artifactId>kafka-streams</artifactId>
      <version>2.0.0</version>
    </dependency>

    <dependency>
    <groupId>org.apache.spark</groupId>
    <artifactId>spark-core_2.11</artifactId>
    <version>2.4.5</version>
    </dependency>

    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-streaming_2.11</artifactId>
      <version>2.4.5</version>
    </dependency>

    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-streaming-kafka-0-10_2.11</artifactId>
      <version>2.4.5</version>
    </dependency>

    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-sql_2.11</artifactId>
      <version>2.4.5</version>
    </dependency>

    <dependency>
      <groupId>com.fasterxml.jackson.core</groupId>
      <artifactId>jackson-databind</artifactId>
      <version>2.6.6</version>
    </dependency>