flink 自定义带超时时间的计数窗口触发器

最新推荐文章于 2023-10-08 19:57:56 发布

普罗米修斯之火

最新推荐文章于 2023-10-08 19:57:56 发布

阅读量1.8k

点赞数 2

分类专栏： flink 文章标签： flink apache 大数据

本文链接：https://blog.csdn.net/WuBoooo/article/details/122881808

版权

flink 专栏收录该内容

10 篇文章 6 订阅

订阅专栏

该博客介绍了一个自定义的Flink窗口触发器——CountTriggerWithTimeout，它结合了计数和超时机制。当窗口内元素达到预设的最大数量或者超过指定的时间限制时，窗口会被触发并清除。示例展示了如何设置处理时间和事件时间，并在窗口中应用这个触发器，以及如何处理窗口内的数据。

摘要由CSDN通过智能技术生成

自定义带超时时间的计数窗口触发器

import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.api.common.state.ReducingState;
import org.apache.flink.api.common.state.ReducingStateDescriptor;
import org.apache.flink.api.common.typeutils.base.LongSerializer;
import org.apache.flink.streaming.api.TimeCharacteristic;
import org.apache.flink.streaming.api.windowing.triggers.Trigger;
import org.apache.flink.streaming.api.windowing.triggers.TriggerResult;
import org.apache.flink.streaming.api.windowing.windows.TimeWindow;


/**
 * 带超时的计数窗口触发器
 */
public class CountTriggerWithTimeout<T> extends Trigger<T, TimeWindow> {


    /**
     * 窗口最大数据量
     */
    private int maxCount;
    /**
     * event time / process time
     */
    private TimeCharacteristic timeType;
    /**
     * 用于储存窗口当前数据量的状态对象
     */
    private ReducingStateDescriptor<Long> countStateDescriptor =
            new ReducingStateDescriptor("counter", new Sum(), LongSerializer.INSTANCE);
    public CountTriggerWithTimeout(int maxCount, TimeCharacteristic timeType) {


        this.maxCount = maxCount;
        this.timeType = timeType;
    }
    private TriggerResult fireAndPurge(TimeWindow window, TriggerContext ctx) throws Exception {
        clear(window, ctx);
        return TriggerResult.FIRE_AND_PURGE;
    }
    @Override
    public TriggerResult onElement(T element, long timestamp, TimeWindow window, TriggerContext ctx) throws Exception {
        ReducingState<Long> countState = ctx.getPartitionedState(countStateDescriptor);
        countState.add(1L);


        if (countState.get() >= maxCount) {
            return fireAndPurge(window, ctx);
        }
        if (timestamp >= window.getEnd()) {
            return fireAndPurge(window, ctx);
        } else {
            return TriggerResult.CONTINUE;
        }
    }


    @Override
    public TriggerResult onProcessingTime(long time, TimeWindow window, TriggerContext ctx) throws Exception {
        if (timeType != TimeCharacteristic.ProcessingTime) {
            return TriggerResult.CONTINUE;
        }


        if (time >= window.getEnd()) {
            return TriggerResult.CONTINUE;
        } else {
            return fireAndPurge(window, ctx);
        }
    }


    @Override
    public TriggerResult onEventTime(long time, TimeWindow window, TriggerContext ctx) throws Exception {
        if (timeType != TimeCharacteristic.EventTime) {
            return TriggerResult.CONTINUE;
        }


        if (time >= window.getEnd()) {
            return TriggerResult.CONTINUE;
        } else {
            return fireAndPurge(window, ctx);
        }
    }


    @Override
    public void clear(TimeWindow window, TriggerContext ctx) throws Exception {
        ReducingState<Long> countState = ctx.getPartitionedState(countStateDescriptor);
        countState.clear();
    }


    /**
     * 计数方法
     */
    class Sum implements ReduceFunction<Long> {


        @Override
        public Long reduce(Long value1, Long value2) throws Exception {
            return value1 + value2;
        }
    }
}

使用案例：

StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);//设置窗口时间类型为系统时间，否则会报错

envStream.keyBy(t-> t)
         .timeWindow(Time.of(5, TimeUnit.SECONDS))//设置超时时间为5秒
         .trigger(new CountTriggerWithTimeout(500, TimeCharacteristic.ProcessingTime))//窗口数据条数为500条
         .process(new CdcMyProcessWindowFunction())//对窗口内的数据类型做一下转换，否则不能调Sink
         .addSink()

/**
 * @Description:对窗口内的数据进行转换后输出
 */
public class CdcMyProcessWindowFunction extends ProcessWindowFunction<String, List<String>, String, TimeWindow> {
    @Override
    public void process(String s, Context context, Iterable<String> elements, Collector<List<String>> out) throws Exception {
        ArrayList<String> sqlArr = Lists.newArrayList(elements);
        if (sqlArr.size() > 0) {
            out.collect(sqlArr);
            sqlArr.clear();
        }
    }
}