Spark-Streaming

 

WordCount案例

添加依赖

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <parent>
        <artifactId>Sparkes</artifactId>
        <groupId>org.example</groupId>
        <version>1.0-SNAPSHOT</version>
    </parent>
    <modelVersion>4.0.0</modelVersion>

    <artifactId>Sparkes-core</artifactId>

    <properties>
        <maven.compiler.source>8</maven.compiler.source>
        <maven.compiler.target>8</maven.compiler.target>
    </properties>

    <dependencies>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-core_2.12</artifactId>
            <version>3.0.0</version>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-streaming_2.12</artifactId>
            <version>3.0.0</version>
        </dependency>
    </dependencies>

</project>

代码

import org.apache.spark.SparkConf
import org.apache.spark.streaming.{Seconds, StreamingContext}

object SparkStreaming {
  def main(args: Array[String]): Unit = {
    // 创建 SparkConf 对象，设置运行模式和应用名称
    val sparkConf = new SparkConf().setMaster("local[*]").setAppName("streaming")
    // 创建 StreamingContext 对象，设置批处理间隔为 3 秒
    val ssc = new StreamingContext(sparkConf, Seconds(3))

    // 从指定的套接字地址接收文本流数据
    val lineStreams = ssc.socketTextStream("node01", 9999)
    // 将每行文本拆分为单词
    val wordStreams = lineStreams.flatMap(_.split(" "))
    // 将每个单词映射为 (单词, 1) 的键值对
    val wordAndOneStreams = wordStreams.map((_, 1))
    // 对相同单词的计数进行累加
    val wordAndCountStreams = wordAndOneStreams.reduceByKey(_ + _)
    // 打印每个批次的单词计数结果
    wordAndCountStreams.print()

    // 启动 StreamingContext
    ssc.start()
    // 等待 StreamingContext 终止
    ssc.awaitTermination()
  }
}