1、代码
import org.apache.log4j.{Level, Logger}
import org.apache.spark.SparkConf
import org.apache.spark.streaming.dstream.{DStream, ReceiverInputDStream}
import org.apache.spark.streaming.{Seconds, StreamingContext}
/**
* @Author caoerbiao
* @Date 2020/6/2 10:47
* @Describe
*/
object HdfsWordCount {
def main(args: Array[String]): Unit = {
Logger.getLogger("org").setLevel(Level.ERROR)
val sparkConf:SparkConf = new SparkConf().setAppName("HdfsWordCount").setMaster("local[2]")
val ssc = new StreamingContext(sparkConf,Seconds(4))
// 监听的hdfs目录只会计算实时放进的文件,不会计算已有文件或者文件夹下一直存在的文件
//todo: 1、监听windows本地文件夹;E:\\data_test
val textFileStream:DStream[String] = ssc.textFileStream("E:\\data_test")
//todo: 2、监听hdfs路径
// val textFileStream:DStream[String] = ssc.textFileStream("hdfs:/node01:8020/data")
val result= textFileStream.flatMap(_.split(