集成分为两个版本:http://spark.apache.org/docs/2.2.0/streaming-kafka-integration.html
import kafka.serializer.StringDecoder
import org.apache.spark.SparkConf
import org.apache.spark.streaming.kafka.KafkaUtils
import org.apache.spark.streaming.{Seconds, StreamingContext}
object streamingKafka {
def main(args: Array[String]): Unit = {
val sparkConf=new SparkConf()
.setMaster("local[2]")
.setAppName("streamingKafka")
val ssc=new StreamingContext(sparkConf,Seconds(5))
val topicsSet = Set("weblogs")
val kafkaParams = Map[String, String]("metadata.broker.list" -> "zeng151.com:9092")
val kfkStream = KafkaUtils.createDirectStream[String, String, StringDecoder, StringDecoder](
ssc, kafkaParams, topicsSet)
// Get the lines, split them into words, count the words and print
val lines = kfkStream.map(_._2)
val words = lines.flatMap(_.split(" "))
val wordCounts = words.map(x => (x, 1L)).reduceByKey(_ + _)
wordCounts.print()
ssc.start()
ssc.awaitTermination()
}
}