Kafka Spark(Consumer) 相关API

BigData_001_Lz

已于 2022-09-15 17:36:53 修改

阅读量103

点赞数

分类专栏：笔记文章标签： kafka spark 分布式

于 2022-09-15 17:33:20 首次发布

本文链接：https://blog.csdn.net/GX_0824/article/details/126876091

版权

笔记专栏收录该内容

41 篇文章 1 订阅

订阅专栏

object MyConsumer {
  def main(args: Array[String]): Unit = {
    val conf: SparkConf = new SparkConf().setAppName("MyConsumer").setMaster("spark://master:7077")
    val ssc = new StreamingContext(conf,Seconds(3))
    //配置连接参数
    val kafkaPara: Map[String, Object] = Map[String,Object](
      ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG -> "master:9092,slave1:9092,slave2:9092",
      ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG -> classOf[StringDeserializer],
      ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG -> classOf[StringDeserializer],
      ConsumerConfig.GROUP_ID_CONFIG -> "test"
    )
    //读取kafka数据创建DStream
    val kafkaDStream: InputDStream[ConsumerRecord[String, String]] =
      KafkaUtils.createDirectStream[String,String](ssc,
        LocationStrategies.PreferConsistent,
        ConsumerStrategies.Subscribe[String, String](Set("reback"), kafkaPara)
      )
    //将每条消息的KV取出
    val valueDStream: DStream[String] = kafkaDStream.map(record => record.value())
    //计算wordcount
    valueDStream.print()
    //开启任务
    ssc.start()
    ssc.awaitTermination()
  }
}