package streamings.studys
import org.apache.spark.SparkConf
import org.apache.spark.streaming.dstream.DStream
import org.apache.spark.streaming.{Seconds, StreamingContext}
/**
* Created by yunxinghai on 3/23/2017.
*/
object Case_2_ReadFromLocalFile {
def main(args: Array[String]): Unit = {
if (args.length > 2){
//val master = url
}else{
val master = "local[6]"
}
val conf = new SparkConf().setAppName("readfile from file local").setMaster("local[6]")
val ssc = new StreamingContext(conf,Seconds(15))
val data: DStream[String] = ssc.textFileStream("C:\\p_not_imprt\\Tesst\\")
val flatData = data.flatMap(_.split(" "))
val words = flatData.map((_,1))
val countEachWord: DStream[(String, Int)] = words.reduceByKey(_+_)
countEachWord.print()
ssc.start()
ssc.awaitTermination()
}
}
Spark Streaming 读取本地文件压文件
最新推荐文章于 2024-04-29 11:12:25 发布