Kafka Spark (Consumer) 相关API （消费指定分区内的数据）

object MyConsumer extends App {
    val conf: SparkConf = new SparkConf().setAppName("MyConsumer").setMaster("local[*]")
    val ssc = new StreamingContext(conf,Seconds(3))
    //配置连接参数
    val kafkaPara: Map[String, Object] = Map[String,Object](
      ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG -> "master:9092,slave1:9092,slave2:9092",
      ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG -> classOf[StringDeserializer],
      ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG -> classOf[StringDeserializer],
      ConsumerConfig.GROUP_ID_CONFIG -> "test"
//      ConsumerConfig.AUTO_OFFSET_RESET_CONFIG -> "latest"
    )
  //指定消费主题下面哪个分区，从哪开始消费
  val offsets = Map[TopicPartition, Long](
    new TopicPartition("reback", 5) -> 11,
    new TopicPartition("reback", 1) -> 22,
    new TopicPartition("reback", 3) -> 5
  )
    //读取kafka数据创建DStream
    val kafkaDStream: InputDStream[ConsumerRecord[String, String]] =
      KafkaUtils.createDirectStream[String,String](ssc,
        LocationStrategies.PreferConsistent,
        ConsumerStrategies.Subscribe[String, String](Set("reback"), kafkaPara,offsets)
      )
    //将每条消息的KV取出
    val valueDStream = kafkaDStream.map(record => ("主题:" + record.topic(), "分区:" + record.partition(), "值:" +record.value()))
    valueDStream.print(300)
    //开启任务
    ssc.start()
    ssc.awaitTermination()
}