pyspark 结构化流读取 两个 kafka主题，累计统计 IP 出现次数 Demo版本

# -*- coding: utf-8 -*-
"""
 @CreateTime :2021/1/5 15:26
 @Author : Liangde
 @Description :

 @Modify:

"""

from pyspark import SparkConf
from pyspark.sql import SparkSession
from conf.setting import KAFKA_CONFIG




"""
    设置任务 常量
"""

TOPIC = KAFKA_CONFIG["TOPIC"]
MAX_OFFSETS_PER_TRIGGER = KAFKA_CONFIG["MAX_OFFSETS_PER_TRIGGER"]
PROCESSING_TIME = KAFKA_CONFIG["PROCESSING_TIME"]

BOOTSTRAP_SERVERS = KAFKA_CONFIG["BOOTSTRAP_SERVERS"]
STARTING_OFFSETS = KAFKA_CONFIG["STARTING_OFFSETS"]


"""

    1、 订阅 kafka 的两个主题

"""

if __name__ == '__main__':

    """
        初始化SparkConf shuffle 分区设为 60
        提交给 yarn 来处理
        建立 spark 对象，并支持启用 hive 的 UDF 函数
    """
    conf = SparkConf() \
        .setAppName('structuredStreamingCleanFFile') \
        .set("spark.sql.shuffle.partitions", 60) \
        .set("stopGracefullyOnShutdown", "true") \
        .set("spark.cleaner.referenceTracking.cleanCheckpoints", "true") \
        .set("spark.executor.memoryOverhead", "2G")

    spark = SparkSession.builder.config(conf=conf).getOrCreate()

    """
        创建 读取 Kafka Source 流
    """

    kafkaSourceFDF = spark.readStream.format("kafka") \
        .option("kafka.bootstrap.servers", BOOTSTRAP_SERVERS) \
        .option("subscribe", TOPIC) \
        .option("startingOffsets", STARTING_OFFSETS) \
        .option("maxOffsetsPerTrigger", MAX_OFFSETS_PER_TRIGGER) \
        .load() \
        .selectExpr("CAST(value AS STRING)", "key") \
        .createOrReplaceTempView("dfTable")

    kafkaSourceFDF2 = spark.sql("""
        with temp as (
            select
                line_list[0] as f_ip,
                line_list[2] as t_ip,
                key as cseq
                from(
                    select  split(value,',') line_list, key from dfTable
                    )
        )

        select t1.ip,t1.cseq,count(1) cnt
        from
        (
            select
                f_ip as ip ,
                cseq
                from temp
            union all
            select
                t_ip as ip,
                cseq
                 from temp
        ) t1
        group by t1.ip,t1.cseq
        order by cnt desc
    """)



    concoleQuery = kafkaSourceFDF2.writeStream \
        .trigger(processingTime='10 seconds') \
        .format("console") \
        .outputMode("complete") \
        .start()

    concoleQuery.awaitTermination()

outputMode(“”) 有三种
Complete Mode (只能选用 complete 模式)
Append Mode (有聚合的不支持 Append 模式)
Update Mode (有聚合的不支持 Update 模式)

setting.py

# -*- coding: utf-8 -*-
"""
 @CreateTime :2020/12/14 18:23
 @Author : Liangde
 @Description :
        存放 kakfa 的相关信息
 @Modify:

"""

KAFKA_CONFIG = {

    'TOPIC': "sip_kfk_fmt_f_r2p6,sip_kfk_fmt_s_r2p6",
    'MAX_OFFSETS_PER_TRIGGER': 1000,
    'PROCESSING_TIME': '5 seconds',


    'BOOTSTRAP_SERVERS': "cdhtest005:9092,cdhtest002:9092,cdhtest003:9092",
    'STARTING_OFFSETS': "latest"

}