SparkStreaming HDFS
以下代码一直读取不了hdfs文件,且没有报错。最后发现是linux时间与本地环境没有进行时间同步。
package org.example.sparkstreaming
import org.apache.log4j.{Level, Logger}
import org.apache.spark.sql.SparkSession
import org.apache.spark.streaming.dstream.DStream
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.{SparkConf, SparkContext}
object SparkStreaming_HDFS {
def main(args: Array[String]): Unit = {
Logger.getLogger("org.apache.spark").setLevel(Level.OFF)
Logger.getLogger("org.eclipse.jetty.server").setLevel(Level.OFF)
val conf=new SparkConf().setAppName("Streaming Word Count OnLine").setMaster("local[2]")
val ssc=new StreamingContext(conf,Seconds(6))
val lines=ssc.textFileStream("hdfs://192.168.172.131:9000/spark/")
val words=lines.flatMap(_.split(" ")).map(word=>(word,1)).reduceByKey(_+_)
words.print()
ssc.start()
ssc.awaitTermination()
}
}