wordcount

最新推荐文章于 2022-09-07 00:48:00 发布

buerba

最新推荐文章于 2022-09-07 00:48:00 发布

阅读量66

点赞数

本文链接：https://blog.csdn.net/buerba/article/details/109635528

版权

package cn.kgc.day0819.test01

import org.apache.spark.SparkConf
import org.apache.spark.streaming.dstream.{DStream, ReceiverInputDStream}
import org.apache.spark.streaming.{Seconds, StreamingContext}

object Demo01 extends App {

//TODO 创建一个spark StreamingContext对象
val conf: SparkConf = new SparkConf().setMaster("local[2]").setAppName("DEMO01")
val ssc = new StreamingContext(conf,Seconds(1))
//TODO 使用spark streaming来进行wordcount

val inputDstream: ReceiverInputDStream[String] = ssc.socketTextStream("hadoop101",9999)
//TODO 对输入的流进行操作
// hadoop spark kafka
val wordDstream: DStream[String] = inputDstream.flatMap(_.split(" "))
val wordAndOneDstream: DStream[(String, Int)] = wordDstream.map((_,1))
val wordcounts: DStream[(String, Int)] = wordAndOneDstream.reduceByKey(_+_)
wordcounts.print()

//TODO 通过start() 启动消息采集和处理
ssc.start()
//TODO 等待程序终止
ssc.awaitTermination()

}