structed streaming 读取kafka数据

1、添加必要的maven依赖

<dependency>
          <groupId>org.scala-lang</groupId>
          <artifactId>scala-library</artifactId>
          <version>2.11</version>
      </dependency>
      <dependency>
          <groupId>org.apache.spark</groupId>
          <artifactId>spark-core_2.11</artifactId>
          <version>2.3.0</version>
      </dependency>
      <dependency>
          <groupId>org.apache.spark</groupId>
          <artifactId>spark-sql_2.11</artifactId>
          <version>2.3.0</version>
      </dependency>
      <dependency>
          <groupId>org.apache.spark</groupId>
          <artifactId>spark-streaming_2.11</artifactId>
          <version>2.3.0</version>
      </dependency>
      <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-sql-kafka-0-10 -->
      <dependency>
          <groupId>org.apache.spark</groupId>
          <artifactId>spark-sql-kafka-0-10_2.11</artifactId>
          <version>2.3.0</version>
      </dependency>
      <!-- https://mvnrepository.com/artifact/org.codehaus.janino/janino -->
      <dependency>
          <groupId>org.codehaus.janino</groupId>
          <artifactId>janino</artifactId>
          <version>3.0.8</version>
      </dependency>

2、完整代码

import conf.{ConfManager, Constans}
import org.apache.log4j.{Level, Logger}
import org.apache.spark.sql.{DataFrame, SparkSession}

object ReadKafka {
def main(args: Array[String]): Unit = {
    Logger.getLogger("org.apache.spark").setLevel(Level.ERROR)
    Logger.getLogger("org.eclipse.jetty.server").setLevel(Level.ERROR)
    Logger.getLogger("org.apache.kafka.clients.consumer").setLevel(Level.ERROR)
    var query:org.apache.spark.sql.streaming.StreamingQuery = null
    val spark = SparkSession
      .builder
      .appName("StructuredStreaming")
      .master("local[2]")
      .getOrCreate()
    val topic = "test"
    val broker = "hadoop01:9092"
    val df = spark
      .readStream
      .format("kafka")
      .option("kafka.bootstrap.servers",broker)
      .option("subscribe",topic)
      .option("startingOffsets", "latest")
      .option("max.poll.records", 100)
      .load()
    import  spark.implicits._
    val word = df.selectExpr("CAST(value AS STRING)").as[String].flatMap(_.split(" "))
    val wordcount = word.groupBy("value").count()
    val q = wordcount
      .writeStream
      .outputMode("complete")
      .format("console")
      .start()
    q.awaitTermination()
  }

}