Watermark 生成

https://ci.apache.org/projects/flink/flink-docs-release-1.12/zh/dev/event_timestamps_watermarks.html#writing-watermarkgenerators

// watermark
        SingleOutputStreamOperator<Tuple2<Event, Long>> upStreamWatermark = upStream.assignTimestampsAndWatermarks(new WatermarkStrategy<Tuple2<Event, Long>>() {
            @Override
            public WatermarkGenerator<Tuple2<Event, Long>> createWatermarkGenerator(WatermarkGeneratorSupplier.Context context) {
                return new WatermarkGenerator<Tuple2<Event, Long>>() {
                    private long maxTimeStamp = Long.MIN_VALUE;

                    @Override
                    public void onEvent(Tuple2<Event, Long> event, long eventTimestamp, WatermarkOutput output) {
                        long maxOutOfOrderness = 0L;
                        maxTimeStamp = Math.max(event.f1, maxTimeStamp);
                        output.emitWatermark(new Watermark(maxTimeStamp - maxOutOfOrderness));
                    }

                    @Override
                    public void onPeriodicEmit(WatermarkOutput output) {
//                        long maxOutOfOrderness = 3000L;
//                        System.out.println(maxTimeStamp - maxOutOfOrderness);
//                        output.emitWatermark(new Watermark(maxTimeStamp - maxOutOfOrderness));
                    }

                };
            }
        }.withIdleness(Duration.ofMinutes(30)).withTimestampAssigner((element, recordTimestamp) -> element.f1));

SingleOutputStreamOperator<Tuple2<Event, Long>> upStreamWatermark = upStream.assignTimestampsAndWatermarks(new AssignerWithPeriodicWatermarks<Tuple2<Event, Long>>() {
            Long currentMaxTimestamp = 0L;
            final Long maxOutOfOrderness = 0L;// 最大允许的乱序时间是5s
            /**
             * 定义生成watermark的逻辑
             * 默认100ms被调用一次
             */
            @Override
            public org.apache.flink.streaming.api.watermark.Watermark getCurrentWatermark() {
                return new org.apache.flink.streaming.api.watermark.Watermark(currentMaxTimestamp - maxOutOfOrderness);
            }

            //定义如何提取timestamp
            @Override
            public long extractTimestamp(Tuple2<Event, Long> element, long previousElementTimestamp) {
                long timestamp = element.f1;
                currentMaxTimestamp = Math.max(timestamp, currentMaxTimestamp);
                //获取当前最大时间戳
//                log.info("currentThreadId:" + Thread.currentThread().getId() + ",key:" + element._1.getMd5() + ",eventtime:[" + element._2 + "],currentMaxTimestamp:[" + currentMaxTimestamp + "|" +
//                        sdf.format(currentMaxTimestamp) + "],watermark:[" + getCurrentWatermark().getTimestamp() + "|" + sdf.format(getCurrentWatermark().getTimestamp()) + "]");
//                System.out.println("当前时间戳:" + getCurrentWatermark().getTimestamp() + ";事件时间:" + timestamp);
                return timestamp;
            }
        });

  • 0
    点赞
  • 0
    收藏
    觉得还不错? 一键收藏
  • 0
    评论
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值