读取Socket数据
import org.apache.spark.SparkContext
import org.apache.spark.sql.streaming.Trigger
import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}
object SokerWOrk {
def main(args: Array[String]): Unit = {
val spark: SparkSession = SparkSession.builder().master("local[*]").appName("SokerWOrk").getOrCreate()
val sc: SparkContext = spark.sparkContext
sc.setLogLevel("WARN")
val dataDF: DataFrame = spark.readStream
.option("host","node01")
.option("port",9999)
.format("socket")
.load()
import spark.implicits._
val dataDS: Dataset[String] = dataDF.as[String]
val wordDS: Dataset[String] = dataDS.flatMap(_.split(" "))
val result: Dataset[Row] = wordDS.groupBy("value").count().sort($"count".desc)
result.writeStream
.format("console")
.outputMode("complete")
.trigger(Trigger.ProcessingTime(0))
.start()
.awaitTermination()
}
}
在其控制台输入 :
反馈数据: