/*
导入依赖
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming-kafka-0-8_2.11</artifactId>
<version>2.1.1</version>
</dependency>
*/
package com.huiyi.deve
import kafka.common.TopicAndPartition
import kafka.message.MessageAndMetadata
import kafka.serializer.StringDecoder
import org.apache.kafka.clients.consumer.ConsumerConfig
import org.apache.spark.SparkConf
import org.apache.spark.streaming.dstream.{DStream, InputDStream}
import org.apache.spark.streaming.kafka.KafkaCluster.Err
import org.apache.spark.streaming.kafka.{HasOffsetRanges, KafkaCluster, KafkaUtils, OffsetRange}
import org.apache.spark.streaming.{Seconds, StreamingContext}
import scala.collection.mutable
object KafkaStreaming {
def main(args: Array[String]): Unit = {
val sparkConf = new SparkConf()
val ssc = new StreamingContext(sparkConf, Seconds(3))
//kakfka参数声明
val group = "mykafka"
val topic = "event_log"
val brokers = "hadoop101:9092,hadoop102:9092,hadoop103:9092"
val deserialization = "org.apache.kafka.common.serialization.StringDeserializer"
//定义kafka参数
val kafkaPara: Map[String, String] = Map[String, String](
ConsumerConfig.GROUP_ID_CONFIG -> group,
ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG -> brokers,
ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG -> deserialization,
ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG -> deserialization
)
//创建KafkaCluster(维护offset)
val kafkaCluster = new KafkaCluster(kafkaPara)
val fromOffsets: Map[TopicAndPartition, Long] = getOffsetFromZookeper(kafkaCluster, group, Set(topic))
//读取kafka创建DStream
val kafkaDStream = KafkaUtils.createDirectStream[String, String, StringDecoder, StringDecoder, String](ssc
, kafkaPara
, fromOffsets
, (x: MessageAndMetadata[String, String]) => x.message())
//处理数据 根据自己的逻辑进行数据处理,比统计从kafka读取的数据 统计单词的个数
val words: DStream[String] = kafkaDStream.flatMap(_.split(" "))
val wordAndOne: DStream[(String, Int)] = words.map((_,1))
val reduceWord: DStream[(String, Int)] = wordAndOne.reduceByKey(_+_)
//打印
reduceWord.print()
//提交offset
offsetToZookeeper(kafkaDStream, kafkaCluster, group)
ssc.start()
ssc.awaitTermination()
}
// 从zk中获取offset
def getOffsetFromZookeper(kafkaCluster: KafkaCluster, group: String, kafkaTopicSet: Set[String]): Map[TopicAndPartition, Long] = {
//创建Map 存储topic和分区对应的offset
val topicAndPartitionOffsetMap = new mutable.HashMap[TopicAndPartition, Long]()
//获取传入topic所有的分区
val topicAndPartition: Either[Err, Set[TopicAndPartition]] = kafkaCluster.getPartitions(kafkaTopicSet)
//如果成果获取到topic所有的分区
if (topicAndPartition.isRight) {
val partitions: Set[TopicAndPartition] = topicAndPartition.right.get
//获取指定分区的offset
val offsetInfo: Either[Err, Map[TopicAndPartition, Long]] = kafkaCluster.getConsumerOffsets(group, partitions)
if (offsetInfo.isLeft) {
//如果没有则offset信息存储为0
for (top <- partitions) {
topicAndPartitionOffsetMap += (top -> 0L)
}
} else {
//如果有offset则存储为offset
val offsets: Map[TopicAndPartition, Long] = offsetInfo.right.get
for ((top, offset) <- offsets) {
topicAndPartitionOffsetMap += (top -> offset)
}
}
}
topicAndPartitionOffsetMap.toMap
}
//提交offset
def offsetToZookeeper(kafkaDStream: InputDStream[String], kafkaCluster: KafkaCluster, group: String): Unit = {
kafkaDStream.foreachRDD {
rdd =>
//获取DStream中的offset
val offsetsList: Array[OffsetRange] = rdd.asInstanceOf[HasOffsetRanges].offsetRanges
//遍历offset信息,并更新到zookeeper中去
for (offset <- offsetsList) {
val topicAndPartition = TopicAndPartition(offset.topic, offset.partition)
val ack: Either[Err, Map[TopicAndPartition, Short]] = kafkaCluster.setConsumerOffsets(group, Map((topicAndPartition, offset.untilOffset)))
if (ack.isLeft) {
println("Erro update offset to kafkaCluster")
} else {
println("successful update offset to kafkaCluser")
}
}
}
}
}