SparkStreaming HDFS监控目录

package org.example.sparkstreaming

import org.apache.log4j.{Level, Logger}
import org.apache.spark.sql.SparkSession
import org.apache.spark.streaming.dstream.DStream
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.{SparkConf, SparkContext}

object SparkStreaming_HDFS {
  def main(args: Array[String]): Unit = {
    Logger.getLogger("org.apache.spark").setLevel(Level.OFF)
    Logger.getLogger("org.eclipse.jetty.server").setLevel(Level.OFF)

    val conf=new SparkConf().setAppName("Streaming Word Count OnLine").setMaster("local[2]")
    val ssc=new StreamingContext(conf,Seconds(6))
    val lines=ssc.textFileStream("hdfs://192.168.172.131:9000/spark/")
    val words=lines.flatMap(_.split(" ")).map(word=>(word,1)).reduceByKey(_+_)
    words.print()
    ssc.start()
    ssc.awaitTermination()
  }
}