HDFSInput

最新推荐文章于 2023-02-05 23:07:30 发布

buerba

最新推荐文章于 2023-02-05 23:07:30 发布

阅读量188

点赞数

本文链接：https://blog.csdn.net/buerba/article/details/109635530

版权

package cn.kgc.day0819.test02

import org.apache.spark.SparkConf
import org.apache.spark.streaming.dstream.DStream
import org.apache.spark.streaming.{Seconds, StreamingContext}

object HDFSInputDStreamDemo extends App {

val conf: SparkConf = new SparkConf().setMaster("local[*]").setAppName("hdfsDemo")
val ssc = new StreamingContext(conf,Seconds(5))

//TODO 创建一个输入流，读取文件系统上的数据
val lines: DStream[String] = ssc.textFileStream("hdfs://hadoop101:9000/data")
val wordcounts: DStream[(String, Int)] = lines.flatMap(_.split(" ")).map((_,1)).reduceByKey(_+_)
wordcounts.print()

ssc.start()
ssc.awaitTermination()

}