一、Flink CEP开发流程
### --- Flink CEP 开发流程:
~~~ DataSource 中的数据转换为 DataStream;watermark、keyby
~~~ 定义 Pattern,并将 DataStream 和 Pattern 组合转换为 PatternStream;
~~~ PatternStream 经过 select、process 等算子转换为 DataStream;
~~~ 再次转换的 DataStream 经过处理后,sink 到目标库。
二、Flink CEP案例一:恶意登录检测
### --- 案例1:恶意登录检测
~~~ 需求:找出5秒内,连续登录失败的账号
~~~ 思路:数据源
new CEPLoginBean(1L, "fail", 1597905234000L),
new CEPLoginBean(1L, "success", 1597905235000L),
new CEPLoginBean(2L, "fail", 1597905236000L),
new CEPLoginBean(2L, "fail", 1597905237000L),
new CEPLoginBean(2L, "fail", 1597905238000L),
new CEPLoginBean(3L, "fail", 1597905239000L),
new CEPLoginBean(3L, "success", 1597905240000L)
### --- 在数据源上做出watermark
### --- 在watermark上根据id分组keyby
### --- 做出模式pattern
Pattern<CEPLoginBean, CEPLoginBean> pattern = Pattern.
<CEPLoginBean>begin("start").where(new IterativeCondition<CEPLoginBean>() {
@Override
public boolean filter(CEPLoginBean value, Context<CEPLoginBean> ctx)
throws Exception {
return value.getLogresult().equals("fail");
}
})
.next("next").where(new IterativeCondition<CEPLoginBean>() {
@Override
public boolean filter(CEPLoginBean value,
Context<CEPLoginBean> ctx) throws Exception {
return value.getLogresult().equals("fail");
}
})
.within(Time.seconds(5));
### --- 在数据流上进行模式匹配
### --- 提取匹配成功的数据
三、编程代码实现
### --- 在pom.xml导入依赖
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-cep_2.12</artifactId>
<version>1.11.1</version>
</dependency>
### --- 编程代码实现
package com.yanqi.mycep;
import org.apache.flink.api.common.eventtime.*;
import org.apache.flink.cep.CEP;
import org.apache.flink.cep.PatternStream;
import org.apache.flink.cep.functions.PatternProcessFunction;
import org.apache.flink.cep.pattern.Pattern;
import org.apache.flink.cep.pattern.conditions.IterativeCondition;
import org.apache.flink.streaming.api.TimeCharacteristic;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.KeyedStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.windowing.time.Time;
import org.apache.flink.util.Collector;
import java.util.List;
import java.util.Map;
public class LoginDemo {
public static void main(String[] args) throws Exception {
/**
* 1、数据源
* 2、在数据源上做出watermark
* 3、在watermark上根据id分组keyby
* 4、做出模式pattern
* 5、在数据流上进行模式匹配
* 6、提取匹配成功的数据
*/
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
env.setParallelism(1);
DataStreamSource<LoginBean> data = env.fromElements(
new LoginBean(1L, "fail", 1597905234000L),
new LoginBean(1L, "success", 1597905235000L),
new LoginBean(2L, "fail", 1597905236000L),
new LoginBean(2L, "fail", 1597905237000L),
new LoginBean(2L, "fail", 1597905238000L),
new LoginBean(3L, "fail", 1597905239000L),
new LoginBean(3L, "success", 1597905240000L)
);
//2、在数据源上做出watermark
SingleOutputStreamOperator<LoginBean> watermarks = data.assignTimestampsAndWatermarks(new WatermarkStrategy<LoginBean>() {
@Override
public WatermarkGenerator<LoginBean> createWatermarkGenerator(WatermarkGeneratorSupplier.Context context) {
return new WatermarkGenerator<LoginBean>() {
long maxTimeStamp = Long.MIN_VALUE;
@Override
public void onEvent(LoginBean event, long eventTimestamp, WatermarkOutput output) {
maxTimeStamp = Math.max(maxTimeStamp, event.getTs());
}
long maxOutOfOrderness = 500L;
@Override
public void onPeriodicEmit(WatermarkOutput output) {
output.emitWatermark(new Watermark(maxTimeStamp - maxOutOfOrderness));
}
};
}
}.withTi