1、sparkstreaming从kafka中读取数据,进行单词统计
首先需要引入依赖,可以参考: https://blog.csdn.net/qq_44665283/article/details/118346847?spm=1001.2014.3001.5501
kafka和sparkstreaming的整合参考官方文档: http://spark.apache.org/docs/latest/streaming-kafka-0-10-integration.html
import org.apache.kafka.clients.consumer.ConsumerRecord
import org.apache.kafka.common.serialization.StringDeserializer
import org.apache.spark.streaming.dstream.{DStream, InputDStream}
import org.apache.spark.streaming.kafka010.{ConsumerStrategies, KafkaUtils, LocationStrategies}
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.{SparkConf, SparkContext}
object _02_SparkStreaming_WordCount_Kafka {
def main(args: Array[String]): Unit = {
//创建sparkStreaming的对象
val conf = new SparkConf().setAppName(this.getClass.getName).setMaster("local[*]")
val context = new SparkContext(conf)
context.setLogLevel("warn") //设置日志的级别
val ssc = new StreamingContext(context, Seconds(5))
ssc.checkpoint("./ck/sparkStream")
//kafkaParams
val kafkaParams = Map[String, Object](
"bootstrap.servers" -> "centos01:9092,centos02:9092,centos03:9092",
"key.deserializer" -> classOf[StringDeserializer],
"value.deserializer" -> classOf[StringDeserializer],
"group.id" -> "g0009",
"auto.offset.reset" -> "earliest" ,//"latest"
//"enable.auto.commit" -> (false: java.lang.Boolean) 先自动提交偏移量,但是自动提交偏移量不能对其进行精准的控制
)
//利用Kafka提供的工具类创建抽象数据集DStream
/*
def createDirectStream[K, V](
ssc: StreamingContext,
locationStrategy: LocationStrategy,
consumerStrategy: ConsumerStrategy[K, V] )
*/
val topics = Array("producer_01")
val stream: DStream[ConsumerRecord[String, String]] = KafkaUtils.createDirectStream(
ssc,
//位置策略.如果本节点有,就从本节点读取
LocationStrategies.PreferConsistent,
//消费策略
ConsumerStrategies.Subscribe[String, String](topics, kafkaParams)
)
//只需要从kafka中读取value值
val lines: DStream[String] = stream.map(consumer => {
consumer.value()
})
//业务逻辑 updateFunc: (Seq[V], Option[S]) => Option[S]
val updateFunc = (seq:Seq[Int],op:Option[Int]) => {
Some(seq.sum + op.getOrElse(0))
}
val reduced: DStream[(String, Int)] = lines.flatMap(_.split(" "))
.map((_, 1)).updateStateByKey(updateFunc)
//.reduceByKey(_ + _)
//触发action
reduced.print()
//开启程序
ssc.start()
//让driver端一直挂起
ssc.awaitTermination()
}
}
2、sparkstreaming从kafka中读取数据并手动提交偏移量
import org.apache.kafka.clients.consumer.ConsumerRecord
import org.apache.kafka.common.serialization.StringDeserializer
import org.apache.spark.rdd.RDD
import org.apache.spark.streaming.dstream.DStream
import org.apache.spark.streaming.kafka010.{CanCommitOffsets, ConsumerStrategies, HasOffsetRanges, KafkaUtils, LocationStrategies, OffsetRange}
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.{SparkConf, SparkContext}
object _03_SparkStreaming_Kafka_CommitOffset {
def main(args: Array[String]): Unit = {
//创建sparkStreaming的对象
val conf = new SparkConf().setAppName(this.getClass.getName).setMaster("local[*]")
val context = new SparkContext(conf)
context.setLogLevel("warn") //设置日志的级别
val ssc = new StreamingContext(context, Seconds(5))
//kafkaParams
val kafkaParams = Map[String, Object](
"bootstrap.servers" -> "centos01:9092,centos02:9092,centos03:9092",
"key.deserializer" -> classOf[StringDeserializer],
"value.deserializer" -> classOf[StringDeserializer],
"group.id" -> "g0010",
"auto.offset.reset" -> "earliest" ,//"latest"
"enable.auto.commit" -> (false: java.lang.Boolean) //先自动提交偏移量,但是自动提交偏移量不能对其进行精准的控制
)
//利用Kafka提供的工具类创建抽象数据集DStream
/*
def createDirectStream[K, V](
ssc: StreamingContext,
locationStrategy: LocationStrategy,
consumerStrategy: ConsumerStrategy[K, V] )
*/
val topics = Array("producer_01")
val stream: DStream[ConsumerRecord[String, String]] = KafkaUtils.createDirectStream(
ssc,
//位置策略.如果本节点有,就从本节点读取
LocationStrategies.PreferConsistent,
//消费策略
ConsumerStrategies.Subscribe[String, String](topics, kafkaParams)
)
//利用foreachRDD获取偏移量
//获取DirectKafkaInputDStream中的偏移量
//DirectKafkaInputDStream或不停地生成KafkaRDD(KafkaRDD的compute方法中调用了kafka消费者的API)
//foreachRDD传入的函数在Driver端周期性的调用
stream.foreachRDD(rdd => {
//类型转换父类
//从kafkaRDD中获取偏移量
//offsetRanges是一个数组类型,数组的长度跟RDD的分区数量一致(KafkaRDD分区的数量与Topic分区数量一致)
//offsetRanges数组中装着OffsetRange对象,里面封装了偏移量相关的信息(topic、partition、offset)
//offsetRanges数组对应下标的偏移量与Kafka分区的下标是一一对应的(即一个topic 的0号分区,对应的偏移量就是数组0号下标的OffsetRange)
//KafkaRDD实现了HasOffsetRanges特质,即只有KafkaRDD中有偏移量
val offsetRanges: Array[OffsetRange] = rdd.asInstanceOf[HasOffsetRanges].offsetRanges
println("******************************************************************")
for (elem <- offsetRanges) {
println("topic: " + elem.topic + ", partition: " + elem.partition +
", fromOffset: " + elem.fromOffset + ", untilOffset: " + elem.untilOffset)
}
//对kafka中的数据进行处理
//编写RDD的api,就是对数据进行处理
//调用RDD的Transformation和Action(调用RDD的方法即Transformation和Action,在Driver调用的)
//传入到Transformation和Action中的函数是在Executor中调用的
val r: RDD[String] = rdd.map(_.value())
r.foreach(println(_))
//自动提交偏移量
//将当前批次的数据处理完成后,将当前批次的偏移量提交到Kafka特殊的topic中(__consumer_offset)
//偏移量也是在Driver端提交的
stream.asInstanceOf[CanCommitOffsets].commitAsync(offsetRanges)
})
//开启程序
ssc.start()
//让driver端一直挂起
ssc.awaitTermination()
}
}