SparkStreaming消费kafka中的数据（0.10版本kafka）

最新推荐文章于 2022-07-07 09:53:13 发布

永不落后于人

最新推荐文章于 2022-07-07 09:53:13 发布

阅读量214

点赞数

分类专栏： spark # kafka

本文链接：https://blog.csdn.net/FlatTiger/article/details/115344855

版权

spark kafka

spark 同时被 2 个专栏收录

45 篇文章 5 订阅

订阅专栏

kafka

15 篇文章 1 订阅

订阅专栏

自动维护offset

kafka0.10版本，topic的offset会维护在kafka的一个系统主题中：也就是__consumer_offsets中。

导入依赖

<dependency>
    <groupId>org.apache.spark</groupId>
    <artifactId>spark-core_2.11</artifactId>
    <version>2.1.1</version>
</dependency>
<dependency>
    <groupId>org.apache.spark</groupId>
    <artifactId>spark-streaming_2.11</artifactId>
    <version>2.1.1</version>
</dependency>
<dependency>
    <groupId>org.apache.spark</groupId>
    <artifactId>spark-streaming-kafka-0-10_2.11</artifactId>
    <version>2.1.1</version>
</dependency>

代码

import org.apache.kafka.clients.consumer.{ConsumerConfig, ConsumerRecord}
import org.apache.kafka.common.serialization.StringDeserializer
import org.apache.spark.SparkConf
import org.apache.spark.streaming.dstream.{DStream, InputDStream}
import org.apache.spark.streaming.kafka010.{ConsumerStrategies, KafkaUtils, LocationStrategies}
import org.apache.spark.streaming.{Seconds, StreamingContext}

/**
  * Author:panghu
  * Date:2021-03-31
  * Description: spark streaming连接kafka-0.10
  * offset维护在kafka的主题 __consumer_offsets 中
  **/
object SparkStreaming01_Kafka010 {
  def main(args: Array[String]): Unit = {
    val conf: SparkConf = new SparkConf().setAppName(this.getClass.getSimpleName).setMaster("local[*]")
    val ssc = new StreamingContext(conf, Seconds(3))
    //配置kafka连接参数
    val kafkaParams: Map[String, Object] = Map[String, Object](
      //kafka节点
      ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG -> "hadoop01:9092,hadoop02:9092,hadoop03:9092",
      //消费者组
      ConsumerConfig.GROUP_ID_CONFIG -> "spark0331",
      //消息的key编解码方式
      //ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG -> "org.apache.kafka.common.serialization.StringDeserializer",
      ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG -> classOf[StringDeserializer],
      //消息的value编解码方式
      ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG -> classOf[StringDeserializer]
    )

    //连接kafka，接收消息 泛型代表消息的kv类型
    val dataDStream: InputDStream[ConsumerRecord[String, String]] = KafkaUtils.createDirectStream[String, String](
      ssc,
      //kafka分区分配策略，大多情况下都是PreferConsistent
      LocationStrategies.PreferConsistent,
      //消费者参数 泛型代表消息的kv类型，参数1是topic，参数2是kafka配置参数
      ConsumerStrategies.Subscribe[String, String](Set("SparkStreaming"), kafkaParams)
    )
    // 取出数据，处理数据
    val resDStream: DStream[(String, Int)] = dataDStream.map(_.value()).flatMap(_.split(" ")).map((_,1)).reduceByKey(_+_)
    resDStream.print()

    //开始采集
    ssc.start()
    //等待采集结束
    ssc.awaitTermination()

  }
}