通过TCP端口9999获取数据,并进行单词计数。
依赖:
ThisBuild / version := "0.1.0-SNAPSHOT"
ThisBuild / scalaVersion := "2.12.12"
libraryDependencies ++= Seq( "org.apache.spark" % "spark-core_2.12" % "3.0.0")
libraryDependencies ++= Seq( "org.apache.spark" % "spark-sql_2.12" % "3.0.0")
libraryDependencies ++= Seq( "org.apache.spark" % "spark-streaming_2.12" % "3.0.0")
libraryDependencies ++= Seq( "mysql" % "mysql-connector-java" % "5.1.30")
lazy val root = (project in file("."))
.settings(
name := "scala-proj"
)
package example3
import org.apache.spark.SparkConf
import org.apache.spark.streaming.dstream.{DStream, ReceiverInputDStream}
import org.apache.spark.streaming.{Duration, Seconds, StreamingContext}
object HelloStreaming01 {
def main(args: Array[String]): Unit = {
val sparkConf = new SparkConf().setMaster("local[*]").setAppName("WordCount")
//5秒采集一次数据
val ssc = new StreamingContext(sparkConf, Seconds(5))
val line: ReceiverInputDStream[String] = ssc
.socketTextStream("xx.xx.xx.xx", 9999)
val words: DStream[String] = line.flatMap(_.split(" "))
val wordTuple: DStream[(String, Int)] = words.map((_, 1))
val result: DStream[(String, Int)] = wordTuple.reduceByKey(_ + _)
result.print()
ssc.start();
ssc.awaitTermination();
}
}
可使用nc来发送数据:
nc -lk 9999