flink 程序入口

qq_39291431
已于 2022-09-30 14:12:59 修改
阅读量650
点赞数
分类专栏： java 文章标签： flink apache java
于 2022-09-30 11:28:59 首次发布
本文链接：https://blog.csdn.net/qq_39291431/article/details/127120334
版权
java 专栏收录该内容
8 篇文章 0 订阅
订阅专栏
import com.flinkRisk.config.FlinkConfig;
import com.flinkRisk.constant.OutSideTag;
import com.flinkRisk.model.*;
import com.flinkRisk.process.BaseBroadcastProcessFunction;
import com.flinkRisk.rule.BaseRule;
import com.flinkRisk.sink.ClickhouseRiskEventSink;
import com.flinkRisk.sink.rocketmq.AlarmEventRocketMqSink;
import com.flinkRisk.source.LocalPosSource;
import com.flinkRisk.source.MysqlRuleSource;
import com.flinkRisk.source.RocketMQPosInfoSource;
import com.flinkRisk.source.rocketmq.RocketMQConfig;
import com.flinkRisk.source.rocketmq.RocketMQSink;
import com.flinkRisk.source.rocketmq.RocketMQSource;
import com.flinkRisk.source.rocketmq.common.selector.DefaultTopicSelector;
import com.flinkRisk.source.rocketmq.common.serialization.AlarmEventSerializationSchema;
import com.flinkRisk.source.rocketmq.common.serialization.VehiclePosInfoDeserializationSchema;
import org.apache.flink.api.common.state.MapStateDescriptor;
import org.apache.flink.api.common.typeinfo.Types;
import org.apache.flink.api.java.utils.ParameterTool;
import org.apache.flink.runtime.state.filesystem.FsStateBackend;
import org.apache.flink.streaming.api.CheckpointingMode;
import org.apache.flink.streaming.api.datastream.*;
import org.apache.flink.streaming.api.environment.CheckpointConfig;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironmen;
import java.io.IOException;
import java.net.URL;
import java.net.URLDecoder;
import java.util.Properties;
@Slf4j
public class Main {

    /**
     * 本地调试模式  部署线上时要改为false
     */
    public static boolean localMode = false;

    /*****************************************本地调试配置开始*****************************************/

    /**
     * 本地需要调试的业务，多个以;号隔开
     */
    public final static String biz = "FatigueDrivingAlarmHandler";

    /**
     * 本地调试时状态存储位置 localMode为true时使用
     */
    private static String stateBackend = "file:///E:/flinkRockDb";

    /**
     * 本地的轨迹文件路径  localMode为true时使用
     */
     private static String localPosDataPath = "E:\\flinkTestData\\1281.txt ";

    /**
     * 轨迹文件开始时间 localMode为true时使用
     */
    private static String startTime = "2022-03-20 05:30:00";
    /**
     * 轨迹文件结束时间 localMode为true时使用
     */
    private static String endTime = "2022-03-20 10:59:59";
    /**
     * 车辆id localMode为true时使用
     */
    private static Integer vehicleId = 9044748;
    /**
     * 车牌 localMode为true时使用
     */
    private static String plate = "05#-13920-D53";
    /*****************************************本地调试配置结束*****************************************/

    /**
     * 线上模式，需要将parameterTool分发给其他算子
     */
    private static ParameterTool parameterTool;
	
	public static void main(String[] args) throws Exception {

        log.info("启动应用...");
        //1、解析命令行参数
        init(args);
        //初始化clickhouse连接

        //checkpoint配置
        String stateBackendDirectory = "";
        //本地调试 状态存储路径
//        if (localMode) {
//            stateBackendDirectory = stateBackend;
//        }
        //线上环境 状态存储路径
        if (!localMode) {
            stateBackendDirectory = parameterTool.getRequired("state.backend.directory");
        }
         //rocketmq车辆定位数据流配置
        Properties consumerProps = new Properties();
        consumerProps.setProperty(RocketMQConfig.NAME_SERVER_ADDR, parameterTool.getRequired("rocketmq.nameserver.address"));
        consumerProps.setProperty(RocketMQConfig.CONSUMER_GROUP, parameterTool.getRequired("rocketmq.consumer.group"));
        consumerProps.setProperty(RocketMQConfig.CONSUMER_TOPIC, parameterTool.getRequired("rocketmq.consumer.topic"));


        //配置流配置


        //2、配置运行环境
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.getConfig().setGlobalJobParameters(parameterTool);
//        线上设置StateBackend
        if(!localMode){
            env.setStateBackend(new FsStateBackend(stateBackendDirectory, true));
//        设置Checkpoint
            CheckpointConfig checkpointConfig = env.getCheckpointConfig();
            env.setBufferTimeout(1000);
            //两分钟超时  Connection reset by peer
            checkpointConfig.setCheckpointTimeout(120000);
            //两个检查点之间至少间隔5秒
            checkpointConfig.setMinPauseBetweenCheckpoints(5000);
            //启用非对齐检查点
            checkpointConfig.enableUnalignedCheckpoints(true);
            //设置多少秒持久化一次快照 单位ms
            checkpointConfig.setCheckpointInterval(parameterTool.getInt("checkpoint.interval"));
            checkpointConfig.setCheckpointingMode(CheckpointingMode.EXACTLY_ONCE);
            checkpointConfig.enableExternalizedCheckpoints(CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION);
        }
        //3、rocketMq车辆定位事件流

        DataStreamSource<VehiclePosInfo> rocketMqStream = null;
        //本地调试
        if (localMode) {
            LocalPosSource localPosSource = new LocalPosSource(localPosDataPath, startTime, endTime, vehicleId, plate);
            rocketMqStream = env.addSource(localPosSource);
        }
          //线上
        if (!localMode) {
            int rocketMqParallelism =  parameterTool.getInt("rocketMqParallelism");
            RocketMQSource<VehiclePosInfo> rocketMqSource = new RocketMQPosInfoSource(new VehiclePosInfoDeserializationSchema(), consumerProps);
            rocketMqStream = env.addSource(rocketMqSource).setParallelism(rocketMqParallelism);
        }
        rocketMqStream.name("rocketMq定位数据流");

        //4、Mysql配置流
        //自定义Mysql Source，周期性地从Mysql中获取配置，并广播出去
        MysqlRuleSource mysqlRuleSource = new MysqlRuleSource();
        DataStreamSource<BaseRule> configStream = env.addSource(mysqlRuleSource);
        configStream.name("mysql规则流");
        /*
          (1) 先建立MapStateDescriptor
          MapStateDescriptor定义了状态的名称、Key和Value的类型。
          这里，MapStateDescriptor中，key是Void类型，value是Map<String, Tuple2<String,Int>>类型。
         */
        final MapStateDescriptor<Long, BaseRule> configDescriptor = new MapStateDescriptor<>("rule", Types.LONG, Types.GENERIC(BaseRule.class));

        /*
          (2) 将配置流广播，形成BroadcastStream
         */
        BroadcastStream<BaseRule> broadcastConfigStream = configStream.broadcast(configDescriptor);
         //5、事件流和广播的配置流连接，形成BroadcastConnectedStream
        BroadcastConnectedStream<VehiclePosInfo, BaseRule> connectedStream = rocketMqStream.keyBy(VehiclePosInfo::getVehicleId).connect(broadcastConfigStream);

        //6、对BroadcastConnectedStream应用process方法，根据配置(规则)处理事件
        int baseBroadcastProcessParallelism = parameterTool.getInt("baseBroadcastProcessParallelism");
        SingleOutputStreamOperator<AlarmEvent> resultStream = connectedStream.process(new BaseBroadcastProcessFunction()).setParallelism(baseBroadcastProcessParallelism);
        //7、输出结果
        //线上环境
        f (!localMode) {
            Properties producerProps = new Properties();
            producerProps.setProperty(RocketMQConfig.NAME_SERVER_ADDR, parameterTool.getRequired("rocketmq.nameserver.address"));
            String topic = parameterTool.getRequired("rocketmq.producer.topic");
            String tag = parameterTool.get("risk-event");
            //没有延迟
            int msgDelayLevel = RocketMQConfig.MSG_DELAY_LEVEL00;
            producerProps.setProperty(RocketMQConfig.MSG_DELAY_LEVEL, String.valueOf(msgDelayLevel));
            //不进行批量发送
            boolean batchFlag = false;
            RocketMQSink<AlarmEvent> alarmEventRocketMQSink = new AlarmEventRocketMqSink(new AlarmEventSerializationSchema(),
                    new DefaultTopicSelector<>(topic,tag), producerProps).withBatchFlushOnCheckpoint(batchFlag);
            int dataToMysqlParallelism = parameterTool.getInt("dataToMysqlParallelism");
            resultStream.addSink(alarmEventRocketMQSink).setParallelism(dataToMysqlParallelism).name("报警事件流");
			}
			//侧向输入，写入clickhouse数据库
        DataStream<AlarmEvent> sideOutput = resultStream.getSideOutput(OutSideTag.RISK_EVENT);
        sideOutput.addSink(new ClickhouseRiskEventSink()).setParallelism(1).name("clickhouse输出流");

        //9、生成JobGraph，并开始执行
        env.execute("风险数据流分析(粤标)");
   }
    /**
     * 配置参数初始化
     *
     * @param args
     * @throws IOException
     */
     private static void init(String[] args) throws IOException {
		 //1、解析命令行参数
        ParameterTool fromArgs = ParameterTool.fromArgs(args);
        //本地模式
        if (localMode) {
            URL url = Thread.currentThread().getContextClassLoader().getResource("config.properties");
            String path = URLDecoder.decode(url.getPath(), "utf-8");
            parameterTool = ParameterTool.fromPropertiesFile(path);
            FlinkConfig.init(parameterTool);
        }
        /**
         * 线上模式
         */
        if (!localMode) {
            parameterTool = ParameterTool.fromPropertiesFile(fromArgs.getRequired("configFile"));
            FlinkConfig.init(parameterTool);
        }
	}
 }