spark-submit
bin/spark-submit --master local[2] --class org.apache.spark.example.streaming.NetworkWordCount --name NetworkWordCount /root/spark-2.4.3-bin-2.6.0-cdh5.15.1/examples/jars/spark-examples_2.11-2.4.3.jar 172.17.198.174 9999
spark-shell
bin/spark-shell --master local[2]
scala> import org.apache.spark.streaming.{Seconds,StreamingContext}
scala> val ssc = new StreamingContext(sc,Seconds(1))
scala> val lines = ssc.socketTextStream("localhost",9999)
scala> val words = lines.flatMap(_.split(" "))
scala> val wordCounts = words.map(x => (x,1)).reduceByKey(_+_)
scala> wordCounts.print
scala> ssc.start
scala> ssc.awaitTermination
import org.apache.spark.SparkConf
import org.apache.spark.streaming.{Seconds, StreamingContext}
object NetworkWC {
def main(args: Array[String]): Unit = {
val sparkConf = new SparkConf().setMaster("local[2]").setAppName("NetworkWordCount")
val ssc = new StreamingContext(sparkConf,Seconds(5))
val lines = ssc.socketTextStream("localhost",9999)
val result = lines.flatMap(_.split(" ")).map((_,1)).reduceByKey(_+_)
result.print
ssc.start()
ssc.awaitTermination()
}
}
import org.apache.spark.SparkConf
import org.apache.spark.streaming.{Seconds, StreamingContext}
//对进入文件夹的新文件触发
object FileWC {
def main(args: Array[String]): Unit = {
val sparkConf = new SparkConf().setMaster("local").setAppName("FileWordCount")
val ssc = new StreamingContext(sparkConf,Seconds(5))
val lines = ssc.textFileStream("file:///Users/eric/Desktop/l2118i/sparktrain/data")
val result = lines.flatMap(_.split(" ")).map((_,1)).reduceByKey(_+_)
result.print()
ssc.start()
ssc.awaitTermination()
}
}