将kafaka跟sparkstreaming进行整合简单的计算任务

object KafkaWordCount {
def main(args: Array[String]): Unit = {
val conf = new SparkConf().setAppName(this.getClass.getSimpleName).setMaster(“local[*]”)
val ssc = new StreamingContext(conf,Seconds(5))
ssc.sparkContext.setLogLevel(“WARN”)
//kafka相关的参数
val kafkaParams = Map[String, Object](
“bootstrap.servers” -> “linux01:9092,linux02:9092,linux03:9092”,
“key.deserializer” -> classOf[StringDeserializer],
“value.deserializer” -> classOf[StringDeserializer],
“groupid” -> “groupId”,
“auto.offset.reset” -> “earliest”,
//消费者自动提交偏移量(写道kafaka特殊的topic中)
“enable.auto.commit” -> (true, java.lang.Boolean)
)

val topics=Array("wordcount")
//将kafaka跟sparkstreaming进行整合 使用官方推荐的直连方式 使用kafka底层的消费API 效率更高
val kafkaDStream: InputDStream[ConsumerRecord[String, String]] = KafkaUtils.createDirectStream(
  //传入streamingcontext
  ssc,
  //位置策略
  LocationStrategies.PreferConsistent,
  //消费策略
  ConsumerStrategies.Subscribe[String, String](topics, kafkaParams)
)
val lines: DStream[String] = kafkaDStream.map(cr => cr.value())
val word: DStream[String] = lines.flatMap(_.split(","))
val wordAndOne: DStream[(String, Int)] = word.map((_,1))
val res = wordAndOne.reduceByKey(_+_)
res.print()

//将任务挂起  一直执行任
ssc.start()
ssc.awaitTermination()

}

}