Java版本Flink异步IO连接MySQL

Java版本Flink异步IO连接MySQL

Flink异步IO简介

Flink异步IO
当与外部系统交互时，例如直接访问外部数据库中的数据，比如在MapFunction执行SQL查询操作，这是一个同步交互的过程：向数据库发送请求，MapFunction等待，直到收到响应。在许多情况下，这种等待占据了函数的绝大多数时间。与数据库的异步交互意味着单个并行函数实例可以同时处理许多请求并同时接收响应。这样，发送其他请求和接收响应就可以占用等待时间。至少，等待时间会分摊到多个请求上。这在绝大多数情况下会使得系统具有更大的吞吐量。

• 同步IO 和 异步IO 对比图
  

创建测试库表

• 创建数据库

create database db_test charset utf8mb4;

• 创建表

create table t_user(
	id int,
	user_name varchar(50),
	age int 
);

• 插入测试数据

INSERT INTO t_user VALUES(1, '张三', 20), (3, '王五', 18), (5, '李四', 21)

代码实现

• maven依赖 (pom.xml)

<dependency>
	<groupId>com.alibaba</groupId>
	<artifactId>druid</artifactId>
	<version>1.2.8</version>
</dependency>
<dependency>
	<groupId>mysql</groupId>
	<artifactId>mysql-connector-java</artifactId>
	<version>8.0.28</version>
</dependency>

• 代码

import com.alibaba.druid.pool.DruidDataSource;
import com.alibaba.druid.pool.DruidPooledConnection;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.datastream.AsyncDataStream;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.async.ResultFuture;
import org.apache.flink.streaming.api.functions.async.RichAsyncFunction;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.util.Collections;
import java.util.concurrent.*;
import java.util.function.Supplier;

public class FlinkMySQLAsync {

    public static void main(String[] args) throws Exception {
        // 获取执行环境
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        // 方便测试，设置并行度为1
        env.setParallelism(1);
        
        // 测试数据
        DataStreamSource<User> userStream = env.fromElements(
                new User("1"),
                new User("2"),
                new User("3"),
                new User("4"),
                new User("5")
        );

        // 获取异步流,将异步函数和流进行整合
        AsyncDataStream.
                unorderedWait(userStream, new MysqlAsyncFunction(), 10, TimeUnit.SECONDS).
                print();

        // 执行任务
        env.execute();
    }

    // 自定义异步函数类
    static class MysqlAsyncFunction extends RichAsyncFunction<User, User> {

        // 连接池
        private DruidDataSource druidDataSource;

        // 线程池
        private ExecutorService executorService;

        @Override
        public void open(Configuration parameters) throws Exception {
            // 创建连接池、配置连接参数
            druidDataSource = new DruidDataSource();
            druidDataSource.setDriverClassName("com.mysql.cj.jdbc.Driver");
            druidDataSource.setUsername("root");
            druidDataSource.setPassword("root");
            druidDataSource.setUrl("jdbc:mysql://localhost:3306/db_test?useUnicode=true&characterEncoding=utf8&serverTimezone=UTC");

            // 创建线程池，用于执行异步操作
            executorService = new ThreadPoolExecutor(5, 15, 1,
                    TimeUnit.MINUTES,
                    new LinkedBlockingDeque<>(100));
        }
        
		// 释放资源
        @Override
        public void close() throws Exception {
        
            // 关闭连接池
            if (druidDataSource != null){
                druidDataSource.close();
            }
            
            // 关闭线程池
            if (executorService != null){
                executorService.shutdown();
            }
        }
        
        // 在此方法中执行异步操作
        @Override
        public void asyncInvoke(User user, ResultFuture<User> resultFuture) throws Exception {

            // 执行异步操作
            Future<User> future = executorService.submit(new Callable<User>() {
                @Override
                public User call() throws Exception {
                    // 从连接池中获取连接
                    DruidPooledConnection connection = druidDataSource.getConnection();
                    // 预编译SQL
                    PreparedStatement preparedStatement = connection.prepareStatement("select * from t_user where id = ?");
                    // 设置参数
                    preparedStatement.setString(1, user.getUserId());
                    // 执行SQL并获取结果
                    ResultSet resultSet = preparedStatement.executeQuery();

                    try {
                        // 封装结果
                        while (resultSet.next()) {
                            String userName = resultSet.getString("user_name");
                            int age = resultSet.getInt("age");
                            user.setUserName(userName);
                            user.setAge(age);
                        }
                    } finally {
                        resultSet.close();
                        preparedStatement.close();
                        connection.close();
                    }
                    return user;
                }
            });

            // 获取异步结果并输出
            CompletableFuture.supplyAsync(new Supplier<User>() {
                @Override
                public User get() {
                    try {
                        return future.get();
                    } catch (InterruptedException | ExecutionException e) {
                        return user;
                    }
                }
            }).thenAccept((User result) -> {
                resultFuture.complete(Collections.singleton(result));
            });

        }
    }

    // 实体类
    static class User {

        private String userId;

        private String userName;

        private int age;

        public User(String id) {
            this.userId = id;
        }

        public String getUserId() {
            return userId;
        }

        public void setUserId(String userId) {
            this.userId = userId;
        }

        public String getUserName() {
            return userName;
        }

        public void setUserName(String userName) {
            this.userName = userName;
        }

        public int getAge() {
            return age;
        }

        public void setAge(int age) {
            this.age = age;
        }

        @Override
        public String toString() {
            return "User{" +
                    "userId='" + userId + '\'' +
                    ", userName='" + userName + '\'' +
                    ", age=" + age +
                    '}';
        }
    }
}

• 查看控制台输出，结果是无序的证明整个过程是异步的

User{userId='4', userName='null', age=0}
User{userId='3', userName='王五', age=18}
User{userId='2', userName='null', age=0}
User{userId='1', userName='张三', age=20}
User{userId='5', userName='李四', age=21}