spark stream:从Kafka中读取数据

amadeus_liu2

于 2024-02-27 16:12:06 发布

阅读量580

点赞数 9

分类专栏： spark kafka 文章标签： spark kafka 大数据

本文链接：https://blog.csdn.net/amadeus_liu2/article/details/136325282

版权

kafka 同时被 2 个专栏收录

37 篇文章 1 订阅

订阅专栏

spark

7 篇文章 0 订阅

订阅专栏

一、添加依赖

ThisBuild / version := "0.1.0-SNAPSHOT"

ThisBuild / scalaVersion := "2.12.12"
libraryDependencies ++= Seq( "org.apache.spark" % "spark-core_2.12" % "3.0.0")
libraryDependencies ++= Seq( "org.apache.spark" % "spark-sql_2.12" % "3.0.0")
libraryDependencies ++= Seq( "org.apache.spark" % "spark-streaming_2.12" % "3.0.0")
libraryDependencies ++= Seq( "org.apache.spark" % "spark-streaming-kafka-0-10_2.12" % "3.0.0")
libraryDependencies ++= Seq( "com.fasterxml.jackson.core" % "jackson-core" % "2.10.1")



libraryDependencies ++= Seq( "mysql" % "mysql-connector-java" % "5.1.30")


lazy val root = (project in file("."))
  .settings(
    name := "scala-proj"
  )

二、demo程序

package example3

import org.apache.kafka.clients.consumer.{ConsumerConfig, ConsumerRecord}
import org.apache.spark.SparkConf
import org.apache.spark.rdd.RDD
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.dstream.{DStream, InputDStream, ReceiverInputDStream}
import org.apache.spark.streaming.kafka010.{ConsumerStrategies, KafkaUtils, LocationStrategies}
import org.apache.spark.streaming.receiver.Receiver
import org.apache.spark.streaming.{Duration, Seconds, StreamingContext}

import scala.collection.mutable
import scala.util.Random

object HelloStreaming04 {
  def main(args: Array[String]): Unit = {
    val sparkConf = new SparkConf().setMaster("local[*]").setAppName("KafkaConsumer")
    val ssc = new StreamingContext(sparkConf, Seconds(5))

    val kafkaPara: Map[String, Object] = Map[String, Object](
      ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG -> "xx.xx.xx.xx:9092", 
      ConsumerConfig.GROUP_ID_CONFIG -> "myGroup",
      "key.deserializer" -> "org.apache.kafka.common.serialization.StringDeserializer",
      "value.deserializer" -> "org.apache.kafka.common.serialization.StringDeserializer"
    )

    val kafkaDataDs: InputDStream[ConsumerRecord[String, String]] = KafkaUtils.createDirectStream[String, String](ssc,
      LocationStrategies.PreferConsistent,
      ConsumerStrategies.Subscribe[String, String](Set("myTopic"), kafkaPara)
    )
    kafkaDataDs.map(_.value()).print()
    ssc.start();
    ssc.awaitTermination();

  }




}