https://ci.apache.org/projects/flink/flink-docs-release-1.12/zh/dev/event_timestamps_watermarks.html#writing-watermarkgenerators
// watermark
SingleOutputStreamOperator<Tuple2<Event, Long>> upStreamWatermark = upStream.assignTimestampsAndWatermarks(new WatermarkStrategy<Tuple2<Event, Long>>() {
@Override
public WatermarkGenerator<Tuple2<Event, Long>> createWatermarkGenerator(WatermarkGeneratorSupplier.Context context) {
return new WatermarkGenerator<Tuple2<Event, Long>>() {
private long maxTimeStamp = Long.MIN_VALUE;
@Override
public void onEvent(Tuple2<Event, Long> event, long eventTimestamp, WatermarkOutput output) {
long maxOutOfOrderness = 0L;
maxTimeStamp = Math.max(event.f1, maxTimeStamp);
output.emitWatermark(new Watermark(maxTimeStamp - maxOutOfOrderness));
}
@Override
public void onPeriodicEmit(WatermarkOutput output) {
// long maxOutOfOrderness = 3000L;
// System.out.println(maxTimeStamp - maxOutOfOrderness);
// output.emitWatermark(new Watermark(maxTimeStamp - maxOutOfOrderness));
}
};
}
}.withIdleness(Duration.ofMinutes(30)).withTimestampAssigner((element, recordTimestamp) -> element.f1));
SingleOutputStreamOperator<Tuple2<Event, Long>> upStreamWatermark = upStream.assignTimestampsAndWatermarks(new AssignerWithPeriodicWatermarks<Tuple2<Event, Long>>() {
Long currentMaxTimestamp = 0L;
final Long maxOutOfOrderness = 0L;// 最大允许的乱序时间是5s
/**
* 定义生成watermark的逻辑
* 默认100ms被调用一次
*/
@Override
public org.apache.flink.streaming.api.watermark.Watermark getCurrentWatermark() {
return new org.apache.flink.streaming.api.watermark.Watermark(currentMaxTimestamp - maxOutOfOrderness);
}
//定义如何提取timestamp
@Override
public long extractTimestamp(Tuple2<Event, Long> element, long previousElementTimestamp) {
long timestamp = element.f1;
currentMaxTimestamp = Math.max(timestamp, currentMaxTimestamp);
//获取当前最大时间戳
// log.info("currentThreadId:" + Thread.currentThread().getId() + ",key:" + element._1.getMd5() + ",eventtime:[" + element._2 + "],currentMaxTimestamp:[" + currentMaxTimestamp + "|" +
// sdf.format(currentMaxTimestamp) + "],watermark:[" + getCurrentWatermark().getTimestamp() + "|" + sdf.format(getCurrentWatermark().getTimestamp()) + "]");
// System.out.println("当前时间戳:" + getCurrentWatermark().getTimestamp() + ";事件时间:" + timestamp);
return timestamp;
}
});