spark-streaming-kafka-0-10_2.12 api

因为更新了scala 2.12 版本 老的kafka api 会报错，翻了半天博客记录一下

package com.sq.sparkstreaming

import org.apache.kafka.clients.consumer.ConsumerRecord
import org.apache.kafka.common.TopicPartition
import org.apache.kafka.common.serialization.StringDeserializer
import org.apache.log4j.{Level, Logger}
import org.apache.spark.streaming.dstream.InputDStream
import org.apache.spark.streaming.kafka010.{ConsumerStrategies, KafkaUtils}
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.streaming.kafka010.LocationStrategies.PreferConsistent

object KafkaToDStream {
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf()
      .setAppName("kafkatospark")
      .setMaster("local[2]")
    val sc = new SparkContext(conf)
    //    Logger.getLogger("org").setLevel(Level.ERROR)
    val ssc = new StreamingContext(sc, Seconds(2))

    val kafkaParams: Map[String, Object] = Map[String, Object](
      "bootstrap.servers" -> "10.37.129.100:9092",
      "key.deserializer" -> classOf[StringDeserializer],
      "value.deserializer" -> classOf[StringDeserializer],
      "group.id" -> "test001",
      "auto.offset.reset" -> "earliest",
      "enable.auto.commit" -> (false: java.lang.Boolean)
    )

    val topics = Array(new TopicPartition("test", 1))

    val topicPartition = Array(new TopicPartition("topic01", 0))

    val offsets: Map[TopicPartition, Long] = Map(new TopicPartition("topic01", 0) -> 2495038L)

    val stream2: InputDStream[ConsumerRecord[String, String]] = KafkaUtils.createDirectStream[String, String](
      ssc,
      PreferConsistent,
      ConsumerStrategies.Assign[String, String](topics, kafkaParams)
    )
    stream2.map(_.value()).print()

    ssc.start()
    ssc.awaitTermination()
    ssc.stop()


  }

}