Flink动态读取Mysql配置程序

写者在写需求时遇到了需要通过动态读取mysql，来控制程序逻辑的场景，此时Flink程序和JavaWeb开发的差异就体现出来了，本来简单的一个全局变量即可处理，但是在分布式的flink中，就不是简单的全局变量了。

1、非Spring环境集成Mybatis

  	<dependency>
            <groupId>org.mybatis</groupId>
            <artifactId>mybatis</artifactId>
            <version>3.5.2</version>
        </dependency>

        <dependency>
            <groupId>com.mysql</groupId>
            <artifactId>mysql-connector-j</artifactId>
            <version>8.0.33</version>
        </dependency>


        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>druid</artifactId>
            <version>1.1.10</version>
        </dependency>

<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE configuration
        PUBLIC "-//mybatis.org//DTD Config 3.0//EN"
        "http://mybatis.org/dtd/mybatis-3-config.dtd">
<configuration>

    <properties resource="application.properties"/>

    <settings>
        <!-- 打印sql日志 -->
        <setting name="logImpl" value="STDOUT_LOGGING"/>
        <setting name="mapUnderscoreToCamelCase" value="true"/>
        <setting name="cacheEnabled" value="false"/>
        <setting name="localCacheScope" value="STATEMENT"/>
    </settings>

    <environments default="development">
        <environment id="development">
            <transactionManager type="JDBC"/>
            <dataSource type="com.soc.pdt.druid.MyDruidDataSourceFactory">
                <property name="driver" value="${jdbc.driver}"/>
                <property name="url" value="${jdbc.url}"/>
                <property name="username" value="${jdbc.username}"/>
                <property name="password" value="${jdbc.password}"/>
                <property name="maxActive" value="5"/>
                <property name="initialSize" value="5"/>
                <property name="minIdle" value="5"/>
                <property name="testWhileIdle" value="true"/>
            </dataSource>
        </environment>
    </environments>
    <mappers>
        <mapper resource="mapper/XXXManagerMapper.xml"/>
    </mappers>
</configuration>

以上是配置及maven信息

 // 根据 mybatis-config.xml 配置的信息得到 sqlSessionFactory
        String resource = "mybatis-config.xml";
        InputStream inputStream;
        try {
            inputStream = Resources.getResourceAsStream(resource);
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
        sqlSessionFactory = new SqlSessionFactoryBuilder().build(inputStream);
        
try {
            while (isRunning) {
                sqlSession = sqlSessionFactory.openSession();
                HashMap<String, String> output = new HashMap<>();
                output.put("job_type", jobType);
                queryMysqlConfig(tenantId, output);
                sqlSession.close();
                ctx.collect(output);
                //每隔多少秒执行一次查询
                Thread.sleep(1000 * 120);
            }
        } catch (Exception ex) {
            log.error("从Mysql获取配置异常...", ex);
        }

以上就是读取mysql数据，这里很简单只是用了一个循环来轮训读取

2、广播配置流

  // 创建广播流
        DataStreamSource<HashMap<String, String>> configStream
                = env.addSource(new MysqlConfigResource(params.get("tenantId"), SystemConfig.JOB_TYPE));
        MapStateDescriptor<Void, Map<String, String>> configDescriptor = new MapStateDescriptor<>("config",
                Types.VOID, Types.MAP(Types.STRING, Types.STRING));
        BroadcastStream<HashMap<String, String>> broadcastStream = configStream.broadcast(configDescriptor);

这里通过构造MysqlConfigResource来进行创建广播流

3、构造mysqlConfigResource

public class MysqlConfigResource extends RichSourceFunction<HashMap<String, String>> implements Serializable {

    private volatile boolean isRunning = true;

    private String tenantId;

    private String jobType;
    private SqlSession sqlSession;
    private SqlSessionFactory sqlSessionFactory;


    public MysqlConfigResource() {
    }


    public MysqlConfigResource(String tenantId, String jobType) {
        this.tenantId = tenantId;
        this.jobType = jobType;
    }

    /**
     * 开始时, 在open()方法中建立连接
     *
     * @param parameters
     */
    @Override
    public void open(Configuration parameters) {
        log.info("open mysql");
        // 根据 mybatis-config.xml 配置的信息得到 sqlSessionFactory
        String resource = "mybatis-config.xml";
        InputStream inputStream;
        try {
            inputStream = Resources.getResourceAsStream(resource);
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
        sqlSessionFactory = new SqlSessionFactoryBuilder().build(inputStream);
    }

    /**
     * 执行完，调用close()方法关系连接，释放资源
     *
     * @throws Exception
     */
    @Override
    public void close() throws Exception {
        super.close();
    }

    /**
     * 调用run()方法获取数据
     *
     * @param ctx
     */
    @Override
    public void run(SourceContext<HashMap<String, String>> ctx) {
        try {
            while (isRunning) {
                sqlSession = sqlSessionFactory.openSession();
                HashMap<String, String> output = new HashMap<>();
                output.put("job_type", jobType);
                queryMysqlConfig(tenantId, output);
                sqlSession.close();
                ctx.collect(output);
                //每隔多少秒执行一次查询
                Thread.sleep(1000 * 120);
            }
        } catch (Exception ex) {
            log.error("从Mysql获取配置异常...", ex);
        }
    }

    /**
     * 取消时，会调用此方法
     */
    @Override
    public void cancel() {
        isRunning = false;
    }

    private void queryMysqlConfig(String tenantId, HashMap<String, String> output) {
        XXXManagerMapper mapper = sqlSession.getMapper(XXXManagerMapper.class);
        
        TypeConfig typeConfig = mapper.getJobConfigByType(type, tenantId);
        output.put("config",
                            SerializedUtils.serializedObj(typeConfig));

    }

}

4、广播流链接事件流

        BroadcastConnectedStream<String, HashMap<String, String>> connect = dataStreamSource.connect(broadcastStream);
        SingleOutputStreamOperator<String> data = connect.process(new DataConvertBroadcastProcessFunction());

5、构造处理方法

@Slf4j
public class DataConvertBroadcastProcessFunction extends BroadcastProcessFunction<String, HashMap<String, String>, String> {

    /**
     * 自定义BroadcastProcessFunction
     * 当事件流中的用户ID在配置中出现时，才对该事件处理, 并在事件中补全用户的基础信息
     * 第一个流(事件流)的数据类型
     * 第二个流(配置流)的数据类型
     * 返回的数据类型
     */

    /**
     * 定义MapStateDescriptor
     */
    MapStateDescriptor<Void, Map<String, String>> configDescriptor
            = new MapStateDescriptor<>("config", Types.VOID, Types.MAP(Types.STRING, Types.STRING));

    /**
     * 读取状态，并基于状态，处理事件流中的数据
     * 在这里，从上下文中获取状态，基于获取的状态，对事件流中的数据进行处理
     *
     * @param value 事件流中的数据
     * @param ctx   上下文
     * @param out   输出零条或多条数据
     */
    @Override
    public void processElement(String value, ReadOnlyContext ctx, Collector<String> out) throws Exception {
        //获取状态
        ReadOnlyBroadcastState<Void, Map<String, String>> broadcastState = ctx.getBroadcastState(configDescriptor);
		//处理
        if (StringUtils.isNotBlank(result)) {
            out.collect(result);
        }
    }

    @Override
    public void processBroadcastElement(HashMap<String, String> value, BroadcastProcessFunction<String, HashMap<String, String>, String>.Context context, Collector<String> collector) throws Exception {
        //获取状态
        BroadcastState<Void, Map<String, String>> broadcastState = context.getBroadcastState(configDescriptor);

        //清空状态
        broadcastState.clear();

        //更新状态
        broadcastState.put(null, value);
    }
}

over