Flink异步IO之HttpClient

 

官方示例：

// This example implements the asynchronous request and callback with Futures that have the
// interface of Java 8's futures (which is the same one followed by Flink's Future)

/**
 * An implementation of the 'AsyncFunction' that sends requests and sets the callback.
 */
class AsyncDatabaseRequest extends RichAsyncFunction<String, Tuple2<String, String>> {

    /** The database specific client that can issue concurrent requests with callbacks */
    private transient DatabaseClient client;

    @Override
    public void open(Configuration parameters) throws Exception {
        client = new DatabaseClient(host, post, credentials);
    }

    @Override
    public void close() throws Exception {
        client.close();
    }

    @Override
    public void asyncInvoke(String key, final ResultFuture<Tuple2<String, String>> resultFuture) throws Exception {

        // issue the asynchronous request, receive a future for result
        final Future<String> result = client.query(key);

        // set the callback to be executed once the request by the client is complete
        // the callback simply forwards the result to the result future
        CompletableFuture.supplyAsync(new Supplier<String>() {

            @Override
            public String get() {
                try {
                    return result.get();
                } catch (InterruptedException | ExecutionException e) {
                    // Normally handled explicitly.
                    return null;
                }
            }
        }).thenAccept( (String dbResult) -> {
            resultFuture.complete(Collections.singleton(new Tuple2<>(key, dbResult)));
        });
    }
}

// create the original stream
DataStream<String> stream = ...;

// apply the async I/O transformation
DataStream<Tuple2<String, String>> resultStream =
    AsyncDataStream.unorderedWait(stream, new AsyncDatabaseRequest(), 1000, TimeUnit.MILLISECONDS, 100);

实际应用：

通过HttpClient访问高德接口

package org.myorg.quickstart;

import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.async.ResultFuture;
import org.apache.flink.streaming.api.functions.async.RichAsyncFunction;
import org.apache.http.HttpResponse;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.nio.client.CloseableHttpAsyncClient;
import org.apache.http.impl.nio.client.HttpAsyncClients;
import org.apache.http.util.EntityUtils;

import java.util.Collections;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Future;
import java.util.function.Supplier;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;

public class AsyncHttpClient extends RichAsyncFunction<String,String> {

    private transient CloseableHttpAsyncClient httpclient;

    @Override
    public void open(Configuration parameters) throws Exception {
        super.open(parameters);
        RequestConfig re = RequestConfig.custom()
                .setSocketTimeout(3000)
                .setConnectTimeout(3000)
                .build();

        HttpAsyncClients.custom()
                .setMaxConnTotal(20)
                .setDefaultRequestConfig(re).build();
        httpclient.start();
    }

    @Override
    public void asyncInvoke(String s, ResultFuture<String> resultFuture) throws Exception {

        String url = "";
        HttpGet httpGet = new HttpGet(url);

        Future<HttpResponse> future = httpclient.execute(httpGet, null);

        CompletableFuture.supplyAsync(new Supplier<String>() {
            @Override
            public String get() {
                try {
                    HttpResponse response = future.get();
                    String province = null;
                    if(response.getStatusLine().getStatusCode() == 200){
                        //获取请求的json字符串
                        String result = EntityUtils.toString(response.getEntity());
                        //转成Json对象
                        JSONObject jsonObject = JSON.parseObject(result);
                        //获取位置信息
                        JSONObject regeocode = jsonObject.getJSONObject("regeocode");
                        if (regeocode != null && !regeocode.isEmpty()){
                            JSONObject address = regeocode.getJSONObject("addressComponent");
                            //获取省市区
                            province = address.getString("province");
                        }

                    }
                    return province;
                } catch (Exception e) {
                    return null;
                }
            }
        }).thenAccept( (String province) -> {
            resultFuture.complete(Collections.singleton(province));
        });
    }




    @Override
    public void close() throws Exception {
        super.close();
        httpclient.close();
    }
}

SingleOutputStreamOperator<String> lines = env.socketTextStream("192.168.1.133", 9999);
SingleOutputStreamOperator<String> result = AsyncDataStream.unorderedWait(lines, new AsyncHttpClient(), 
       0, TimeUnit.MILLISECONDS, 10);