spark 2.3官网提供的读取kafka流数据

package com.ultimate.music.streaming.tango

import java.sql.PreparedStatement

import com.ultimate.music.util.PropertyUtil._
import com.ultimate.music.util.{PropertyUtil}
import org.apache.kafka.clients.consumer.ConsumerRecord
import org.apache.spark.rdd.RDD
import org.apache.spark.streaming.kafka010.LocationStrategies._
import org.apache.spark.streaming.kafka010.{CanCommitOffsets, ConsumerStrategies, HasOffsetRanges, KafkaUtils}
import org.apache.spark.streaming.{Seconds, StreamingContext}

object Test {
  val streamingInterval = PropertyUtil.getKey(STREAMING_INTERVAL).toLong
  val spark = SparkSession
    .builder()
    .appName("streamingApp")
    .config(sparkConf)
    .getOrCreate()

  //lazy val sc = new SparkContext(sparkConf)
  val sc = spark.sparkContext
  lazy val ssc = new StreamingContext(sc, Seconds(streamingInterval))

  def main(args: Array[String]): Unit = {
    val messages1 = KafkaUtils.createDirectStream[String, String](ssc, PreferConsistent, ConsumerStrategies.Subscribe[String, String]( PropertyUtil.getKey("topic.t1.api"):: Nil, kafkaParams))
  
    messages1.foreachRDD(rdd => {
      val offsetRanges = rdd.asInstanceOf[HasOffsetRanges].offsetRanges
      if (!rdd.isEmpty()) {
        //do somthing else ...
       // km.persistOffsets[String, String](rdd)
        messages1.asInstanceOf[CanCommitOffsets].commitAsync(offsetRanges)
      }
  
    })

    ssc.start()
    //等待结束
    ssc.awaitTermination()
 
  }

}