object WCKafka extends App{
System.setProperty("hadoop.home.dir", "E:\\software\\bigdate\\hadoop-2.6.0-cdh5.15.0\\hadoop-2.6.0-cdh5.15.0")
val conf= new SparkConf();
conf.setMaster("local[2]")//一个线程用于读数据,一个用于处理数据
conf.setAppName("WordCountStreaming")//在yarn资源管理器监控
val sc= new SparkContext(conf);
val streamingContext= new StreamingContext(sc,Seconds(5))
/*
* ssc: StreamingContext,
zkQuorum: String,
groupId: String,
topics: JMap[String, JInt],
storageLevel: StorageLevel*/
val sourceDS=
KafkaUtils.createStream(streamingContext, "wangfutai:2181", "groupID", Map(("stremaingTest2"->1)))
sourceDS.flatMap(_._2.split(" "))
.map((_,1))
.reduceByKey(_+_).print()
//开启spark streaming任务
streamingContext.start()
//需要等待任务执行
//当调用这个函数,整个任务会在后台不停运行
streamingContext.awaitTermination()
}