Flink系列--cumulate累计窗口的使用

最新推荐文章于 2024-12-05 23:03:02 发布

似水_逆行

最新推荐文章于 2024-12-05 23:03:02 发布

阅读量623

点赞数 6

分类专栏： Flink 文章标签： flink

本文链接：https://blog.csdn.net/qq_43462685/article/details/141459093

版权

Flink 专栏收录该内容

21 篇文章 0 订阅

订阅专栏

背景：

　　一个简单的场景，我们要统计数据通过flink一天的入库数据量，但是我想每10s中看一下当前已经入库的数据量。

　　flink cumulate累计窗口就是解决类似问题的，在数据大屏场景应用广泛

tips：窗口触发输出有watermark推进，一种时数据处理时间，也可以是事件事件

demo

模拟数据输入，然后通过cumulate统计1分钟累计销售额，每10s输出结果

package com.test.flink.table.cumulate.producer.consumer;

import org.apache.flink.table.api.EnvironmentSettings;
import org.apache.flink.table.api.TableEnvironment;

/**
 * 场景示例：
 * 1、每天统计实时收入，做实时大屏统计， 每秒更新结果
 * 延伸场景：
 * 1、23：00 ~ 第二天 01：00  offset场景
 * 2、其他聚合场景
 */
public class TestCumulate {


    public static void main(String[] args) {
        EnvironmentSettings settings = EnvironmentSettings.inStreamingMode();
        TableEnvironment tEnv = TableEnvironment.create(settings);
        tEnv.getConfig().addJobParameter("parallelism.default","1");

//        DataTypes.TIMESTAMP(3)
        tEnv.executeSql("CREATE TABLE consumer (\n" +
                "    consumerTimestamp  TIMESTAMP(3),\n" +
                "    product      STRING,\n" +
                "    price DOUBLE" +
                ",\n" +
                "    WATERMARK FOR consumerTimestamp AS consumerTimestamp - INTERVAL '0' SECOND\n" +
                ") WITH (\n" +
                "    'connector' = 'consumer'" +
                ")");



        tEnv.executeSql("CREATE TABLE print1 (\n" +
                "    window_start  TIMESTAMP(3),\n" +
                "    window_end  TIMESTAMP(3),\n" +
                "    sumPrice     Double ,\n" +
                "    total BIGINT" +
                ") WITH (\n" +
                "   'connector'  = 'print',\n" +
                "   'sink.parallelism'='1'\n" +
                ")");

        tEnv.executeSql("insert into print1 " +
                "select " +
                " window_start," +
                " window_end," +
                //sum统计值，truncate 把42.01323 转成 42.0 cast转成bigint 转成42
//                "CAST(TRUNCATE(sum(price),2) as BIGINT )as sumPrice," +
                "TRUNCATE(sum(price),2) as sumPrice," +
                "count(product) as total " +
                "from TABLE(CUMULATE(" +
                "TABLE consumer," +
                "DESCRIPTOR(consumerTimestamp)," +
                "INTERVAL '10' SECOND," +
                "INTERVAL '1' MINUTES" +
                "))"
               + " group by window_start,window_end"
        );


//
//        tEnv.executeSql("CREATE TABLE print (\n" +
//                "    window_start  BIGINT,\n" +
//                "    window_end  BIGINT,\n" +
//                "    sumPrice     DOUBLE ,\n" +
//                "    total BIGINT" +
//                ") WITH (\n" +
//                "   'connector'  = 'print',\n" +
//                "   'sink.parallelism'='1'\n" +
//                ")");
//
//
//
//        tEnv.executeSql("insert into print " +
//                "select " +
//                "UNIX_TIMESTAMP(CAST(window_start AS STRING)) * 1000 as window_start," +
//                "UNIX_TIMESTAMP(CAST(window_end AS STRING)) * 1000 as window_end," +
//                "TRUNCATE(sum(price),0) as sumPrice," +
//                "count(product) as total " +
//                "from TABLE(CUMULATE(" +
//                "TABLE consumer," +
//                "DESCRIPTOR(consumerTimestamp)," +
//                "INTERVAL '10' SECOND," +
//                "INTERVAL '1' MINUTES" +
//                ")) group by window_start,window_end");
    }
}

数据源实现

import org.apache.flink.configuration.ConfigOption;
import org.apache.flink.table.connector.source.DynamicTableSource;
import org.apache.flink.table.factories.DynamicTableSourceFactory;

import java.util.HashSet;
import java.util.Set;


public class ConsumerDataGenFactory implements DynamicTableSourceFactory {
    @Override
    public DynamicTableSource createDynamicTableSource(Context context) {
        return new ConsumerDataTableSource();
    }

    @Override
    public String factoryIdentifier() {
        return "consumer";
    }

    @Override
    public Set<ConfigOption<?>> requiredOptions() {
        return new HashSet<>();
    }

    @Override
    public Set<ConfigOption<?>> optionalOptions() {
        return new HashSet<>();
    }
}

import org.apache.flink.streaming.api.functions.source.SourceFunction;
import org.apache.flink.table.data.GenericRowData;
import org.apache.flink.table.data.StringData;
import org.apache.flink.table.data.TimestampData;

import java.time.LocalDateTime;
import java.util.Random;


public class ConsumerDataSourceFunction implements SourceFunction {

    private final String[] product = new String[]{
            "A", "B", "C", "D", "E"
    };

    private final Double[] price = new Double[]{
            10.01d,20.01d,30.01d,40.01d,50.01d
    };

    @Override
    public void run(SourceContext ctx) throws Exception {
        /**
         * 产生几条测试数据，用来测试 cumulate 窗口
         * 数据结构
         * 消费时间   产品名称     消费金额
          */
        GenericRowData row = null;
        Random random = new Random();
        long startTime = System.currentTimeMillis();
        while (true) {
            if((System.currentTimeMillis() - startTime) > 1000) {
                startTime = System.currentTimeMillis();
                row = new GenericRowData(3);
                row.setField(0, TimestampData.fromLocalDateTime(
                        LocalDateTime.now()));
                row.setField(1, StringData.fromString(product[random.nextInt(product.length)]));
                row.setField(2, price[random.nextInt(price.length)]);
                ctx.collect(row);
            }
        }
    }

    @Override
    public void cancel() {
    }
}


import org.apache.flink.table.connector.ChangelogMode;
import org.apache.flink.table.connector.source.DynamicTableSource;
import org.apache.flink.table.connector.source.ScanTableSource;
import org.apache.flink.table.connector.source.SourceFunctionProvider;


public class ConsumerDataTableSource implements ScanTableSource {

    @Override
    public ChangelogMode getChangelogMode() {
        return ChangelogMode.insertOnly();
    }

    @Override
    public ScanRuntimeProvider getScanRuntimeProvider(ScanContext runtimeProviderContext) {
        return SourceFunctionProvider.of(new ConsumerDataSourceFunction(),true);
    }

    @Override
    public DynamicTableSource copy() {
        return new ConsumerDataTableSource();
    }

    @Override
    public String asSummaryString() {
        return this.getClass().getSimpleName();
    }
}

maven依赖

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <parent>
        <artifactId>FlinkTableApiTest</artifactId>
        <groupId>org.example</groupId>
        <version>1.0-SNAPSHOT</version>
    </parent>
    <modelVersion>4.0.0</modelVersion>

    <artifactId>cumulate</artifactId>

    <properties>
        <maven.compiler.source>8</maven.compiler.source>
        <maven.compiler.target>8</maven.compiler.target>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    </properties>

    <dependencies>
                <dependency>
                    <groupId>org.slf4j</groupId>
                    <artifactId>slf4j-simple</artifactId>
                </dependency>
        <dependency>
            <groupId>org.junit.jupiter</groupId>
            <artifactId>junit-jupiter-api</artifactId>
            <scope>test</scope>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-statebackend-rocksdb</artifactId>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-java</artifactId>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-runtime</artifactId>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-streaming-java</artifactId>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-clients</artifactId>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-runtime-web</artifactId>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-kafka</artifactId>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-api-java</artifactId>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-base</artifactId>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-planner_2.12</artifactId>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-state-processor-api</artifactId>
        </dependency>

        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <scope>test</scope>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-core</artifactId>
        </dependency>

    </dependencies>
</project>

spi