Flink--AsyncI/O

使用异步IO查询高德地图关联数据

/**
 * 封装数据的对象
 */
public class OrderBean {
    public String oid;

    public String cid;

    public Double money;

    public Double longitude;

    public Double latitude;

    public String province;

    public String city;

    public OrderBean(){}
    public OrderBean(String oid, String cid, Double money, Double longitude, Double latitude) {
        this.oid = oid;
        this.cid = cid;
        this.money = money;
        this.longitude = longitude;
        this.latitude = latitude;
    }

    public static OrderBean of(String oid, String cid, Double money, Double longitude, Double latitude) {
        return new OrderBean(oid, cid, money, longitude, latitude);
    }

    @Override
    public String toString() {
        return "OrderBean{" + "oid='" + oid + '\'' + ", cid='" + cid + '\'' + ", money=" + money + ", longitude=" + longitude + ", latitude=" + latitude + ", province='" + province + '\'' + ", city='" + city + '\'' + '}';
    }
}

public class AsyncIODemo {

    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        //测试数据
        //{"oid": "o1000", "cid": "c10", "money": 99.99, "longitude": 116.413467, "latitude": 39.908072}
        //{"oid": "o1001", "cid": "c11", "money": 99.99, "longitude": 115.413467, "latitude": 39.908072}
        //{"oid": "o1000", "cid": "c10", "money": 99.99, "longitude": 116.413467, "latitude": 39.908072}
        //{"oid": "o1001", "cid": "c11", "money": 99.99, "longitude": 116.413467, "latitude": 39.908072}
        //{"oid": "o1000", "cid": "c10", "money": 99.99, "longitude": 116.413467, "latitude": 39.908072}
        //{"oid": "o1001", "cid": "c11", "money": 99.99, "longitude": 116.413467, "latitude": 39.908072}
        DataStreamSource<String> lines = env.socketTextStream("linux01", 7777);
        //使用异步IO进行查询关联信息
        //unorderedWait发送请求返回的结果与发送请求的先后顺序可以一致
        //orderedWait发送请求返回的结果与发送请求的先后顺序一致
        SingleOutputStreamOperator<OrderBean> result = AsyncDataStream.unorderedWait(lines, new AsyncHttpQueryFunction(), 2000, TimeUnit.MILLISECONDS, 10);

        result.print();

        env.execute();


    }

    private static class AsyncHttpQueryFunction extends RichAsyncFunction<String,OrderBean>{
        private String key = "30b53366c61ebb8385081113dd875e49";
        private transient CloseableHttpAsyncClient httpAsyncClient;
        @Override
        public void open(Configuration parameters) throws Exception {
            //获取连接
            RequestConfig requestConfig = RequestConfig.custom()
                    .setConnectTimeout(3000)    //设置连接的超时时间
                    .setSocketTimeout(3000)     //设置Socket超时时间
                    .build();
             httpAsyncClient = HttpAsyncClients.custom()
                    .setMaxConnTotal(20)        //设置最大请求的个数
                    .setDefaultRequestConfig(requestConfig)
                    .build();
             httpAsyncClient.start();
        }

        @Override
        public void asyncInvoke(String input, ResultFuture<OrderBean> resultFuture) throws Exception {

            try {
                //将输入的数据转换成Bean
                OrderBean orderBean = JSON.parseObject(input, OrderBean.class);
                String latitude = orderBean.latitude;
                String longitude = orderBean.longitude;
                String uri = "https://restapi.amap.com/v3/geocode/regeo?output=json&location="+longitude+","+latitude+"&key="+key;
                HttpGet httpGet = new HttpGet(uri);
                Future<HttpResponse> future = httpAsyncClient.execute(httpGet, null);

                CompletableFuture.supplyAsync(new Supplier<OrderBean>() {

                    @Override
                    public OrderBean get() {
                        try {
                            HttpResponse response = future.get();

                            if (response.getStatusLine().getStatusCode() == 200) {

                                //将获取的数据转成字符串
                                String result = EntityUtils.toString(response.getEntity());
                                //转换成json对象
                                JSONObject jsonObject = JSON.parseObject(result);
                                JSONObject regeocode = jsonObject.getJSONObject("regeocode");
                                if (regeocode != null && !regeocode.isEmpty()) {
                                    JSONObject addressComponent = regeocode.getJSONObject("addressComponent");
                                    orderBean.province = addressComponent.getString("province");
                                    orderBean.city = addressComponent.getString("city");
                                    orderBean.district = addressComponent.getString("district");
                                }
                            }
                            return orderBean;

                        }catch (Exception e){
                            return orderBean;
                        }
                    }
                }).thenAccept(new Consumer<OrderBean>() {
                    @Override
                    public void accept(OrderBean orderBean) {
                        resultFuture.complete(Collections.singleton(orderBean));
                    }
                });
            }catch (Exception e){
                e.printStackTrace();
            }
        }

        @Override
        public void close() throws Exception {
            //释放资源
            httpAsyncClient.close();
        }
    }
}

使用异步IO关联MySQL中的数据

public class AsyncQueryFromMySQLDemo {
    public static void main(String[] args) throws Exception {

        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

        DataStreamSource<String> lines = env.socketTextStream("linux01", 7777);
        int maxConnTotal =20 ;
        SingleOutputStreamOperator<Tuple2<String, String>> result = AsyncDataStream.orderedWait(lines, new AsyncQueryMySQL(maxConnTotal), 1000, TimeUnit.MILLISECONDS, 20);

        result.print();

        env.execute();


    }


    private static class AsyncQueryMySQL extends RichAsyncFunction<String, Tuple2<String,String>>{
        private transient ExecutorService pool;
        private transient DruidDataSource druidDataSource;
        private int maxConnTotal ;

        public AsyncQueryMySQL(int maxConnTotal) {
            this.maxConnTotal = maxConnTotal;
        }

        @Override
        public void open(Configuration parameters) throws Exception {
            //创建线程池
            pool = Executors.newFixedThreadPool(maxConnTotal);
            //创建MySQL连接池
            druidDataSource = new DruidDataSource();
            druidDataSource.setUrl("jdbc:mysql://localhost:3306/test");
            druidDataSource.setUsername("root");
            druidDataSource.setPassword("root");
            druidDataSource.setMaxActive(maxConnTotal);

        }

        @Override
        public void asyncInvoke(String id, ResultFuture<Tuple2<String, String>> resultFuture) throws Exception {
            Future<String> future = pool.submit(new Callable<String>() {
                @Override
                public String call() throws Exception {
                    return query(id);
                }
            });

            CompletableFuture.supplyAsync(new Supplier<String>() {
                @Override
                public String get() {
                    try {
                        return future.get();
                    } catch (Exception e) {
                        return null;
                    }
                }
            }).thenAccept(new Consumer<String>() {
                @Override
                public void accept(String name) {
                    resultFuture.complete(Collections.singleton(Tuple2.of(id,name)));
                }
            });

        }

        public String query(String id) throws SQLException {
            String name = "未知";
            DruidPooledConnection conn = null;
            PreparedStatement preparedStatement = null;
            ResultSet rs = null;
            try {
                //获取连接
                conn = druidDataSource.getConnection();
                String sql ="select name from tb_data where id = ?";
                //获取预编译对象
                preparedStatement = conn.prepareStatement(sql);
                preparedStatement.setString(1,id);
                //执行查询
                rs = preparedStatement.executeQuery();
                while (rs.next()){
                    name = rs.getString("name");
                }
            }finally {
                if (rs != null){
                   rs.close();
                }

                if (preparedStatement != null){
                   preparedStatement.close();
                }

                if (conn != null){
                    conn.close();
                }
            }
            return name;
        }

        @Override
        public void close() throws Exception {
            //关闭连接
            pool.shutdown();
            druidDataSource.close();
        }
    }
}