● 代码 ,kafka不会重复消费,自己会记住偏移量
即使设置了从头开始消费,也不会消费已消费的数据
package com.ws.streaming
import org.apache.spark.SparkConf
import org.apache.spark.streaming.dstream.{DStream, ReceiverInputDStream}
import org.apache.spark.streaming.kafka.KafkaUtils
import org.apache.spark.streaming.{Seconds, StreamingContext}
object KafkaWordCount {
def main(args: Array[String]): Unit = {
val conf = new SparkConf().setMaster("local[*]").setAppName("KafkaWordCount")
val ssc = new StreamingContext(conf,Seconds(5))
val zk = "hadoop-01:2181,hadoop-02:2181,hadoop-03:2181"
val groupId = "g1"
val topic = Map[String,Int]("ws"->1)
//创建kafka DStream
val kafakaDStream: ReceiverInputDStream[(String, String)] = KafkaUtils.createStream(ssc,zk,groupId,topic)
//ReceiverInputDStream[(String, String)]是一个元组,key是写入的key,value是实际写入的数据
val data: DStream[String] = kafakaDStream.map(_._2)
val result: DStream[(String, Int)] = data.flatMap(_.split(" ")).map((_,1)).reduceByKey(_+_)
result.print()
ssc.start()
ssc.awaitTermination()
}
}
● 启动一个kafka命令行生产者
[root@hadoop-03 kafka_2.11-0.8.2.2]# ./bin/kafka-console-producer.sh --broker-list hadoop-01:9092,hadoop-02:9092,hadoop-03:9092 -topic ws
[2018-09-15 18:09:01,454] WARN Property topic is not valid (kafka.utils.VerifiableProperties)
hello jerert i love you m^Hbb zhang wensi
yyyyy yyyyy yyyy y y y yy