针对高并发场景下固定地址HTTP请求的优化方案

背景

最近项目中因为开发工作基本完成，再使用jmeter并发压测检查接口质量时，发现其中有个方法的相应时间居然高达4000+ms，经过排查发现居然是由于该方法中使用了HTTP请求调用了外部系统的依赖方法导致的。

问题分析

由于项目场景比较特殊，因此调用的外部接口方法IP和端口都比较固定。由此我想到是否可以通过共用链接的形式来减少HTTP中建立链接和销毁连接的性能和时间消耗。

改进方案

之前的HTTP调用直接使用了HttpClientUtil工具包下的POST方法，但在这种方案会导致在高密度的请求下频繁的创建和销毁连接（注：HTTP还是由TCP/IP来实现的）导致速度较慢。经过查询和参考

https://www.jianshu.com/p/c852cbcf3d68

得知JDK提供了CloseableHttpClient和PoolingHttpClientConnectionManager这两个类来提供HTTP连接池经行管理HTTP请求因此降低了建立链接和销毁连接的损耗。

代码实现

package com.comtop.dop.bdap.appservice;


import java.io.IOException;
import java.io.InputStream;
import java.io.InterruptedIOException;
import java.io.UnsupportedEncodingException;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TimerTask;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;

import javax.net.ssl.SSLException;
import javax.net.ssl.SSLHandshakeException;

import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.HttpEntityEnclosingRequest;
import org.apache.http.HttpHost;
import org.apache.http.HttpRequest;
import org.apache.http.NameValuePair;
import org.apache.http.NoHttpResponseException;
import org.apache.http.client.HttpRequestRetryHandler;
import org.apache.http.protocol.HTTP;
import org.apache.http.protocol.HttpContext;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpRequestBase;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.conn.ConnectTimeoutException;
import org.apache.http.conn.routing.HttpRoute;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.LayeredConnectionSocketFactory;
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.message.BasicNameValuePair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.comtop.cip.json.JSON;
import com.comtop.cip.json.JSONObject;
import com.comtop.dop.api.engine.dag.DagController;
import com.google.gson.Gson;
import com.google.gson.JsonObject;

import comtop.soa.org.apache.cxf.helpers.IOUtils;

public class HttpConnectionPoolUtil {
	/** 
	 * 日志对象 
	 */
	
	
    private static Logger logger = LoggerFactory.getLogger(HttpConnectionPoolUtil.class);

    private static final int CONNECT_TIMEOUT = 3000;// 设置连接建立的超时时间为30s
    private static final int SOCKET_TIMEOUT = 5000;
    private static final int MAX_CONN = 100; // 最大连接数
    private static final int Max_PRE_ROUTE = 60; // 路由最大连接数
    private static final int MAX_ROUTE =60;
    private static CloseableHttpClient httpClient; // 发送请求的客户端单例
    private static PoolingHttpClientConnectionManager manager; //连接池管理类
    private static ScheduledExecutorService monitorExecutor;

    private final static Object syncLock = new Object(); // 相当于线程锁,用于线程安全

    /**
     * 对http请求进行基本设置
     * @param httpRequestBase http请求
     */
    private static void setRequestConfig(HttpRequestBase httpRequestBase){
        RequestConfig requestConfig = RequestConfig.custom().setConnectionRequestTimeout(CONNECT_TIMEOUT)
                .setConnectTimeout(CONNECT_TIMEOUT)
                .setSocketTimeout(SOCKET_TIMEOUT).build();

        httpRequestBase.setConfig(requestConfig);
    }

    public static CloseableHttpClient getHttpClient(String url){
    	long startTime=System.currentTimeMillis();
        if (httpClient == null){
            //多线程下多个线程同时调用getHttpClient容易导致重复创建httpClient对象的问题,所以加上了同步锁
            synchronized (syncLock){
                if (httpClient == null){
                	String hostName = url.split("/")[2];
                	int port = 80;
                	if (hostName.contains(":")){
                		String[] args = hostName.split(":");
                		hostName = args[0];
                		port = Integer.parseInt(args[1]);
                	}
                    httpClient = createHttpClient(hostName, port);
                    //开启监控线程,对异常和空闲线程进行关闭
                    monitorExecutor = Executors.newScheduledThreadPool(1);
                    monitorExecutor.scheduleAtFixedRate(new TimerTask() {
                        @Override
                        public void run() {
                            //关闭异常连接
                            manager.closeExpiredConnections();
                            //关闭1s空闲的连接
                            manager.closeIdleConnections(1000, TimeUnit.MILLISECONDS);
                            //logger.info("close expired and idle for over 1s connection");
                        }
                    }, 20, 20, TimeUnit.MILLISECONDS); 
                }
            }
        }       
        return httpClient;
    }

    /**
     * 根据host和port构建httpclient实例
     * @param host 要访问的域名
     * @param port 要访问的端口
     * @return
     */
    public static CloseableHttpClient createHttpClient(String host, int port){
        ConnectionSocketFactory plainSocketFactory = PlainConnectionSocketFactory.getSocketFactory();
        LayeredConnectionSocketFactory sslSocketFactory = SSLConnectionSocketFactory.getSocketFactory();
        Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory> create().register("http", plainSocketFactory)
                .register("https", sslSocketFactory).build();

        manager = new PoolingHttpClientConnectionManager(registry);
        //设置连接参数
        manager.setMaxTotal(MAX_CONN); // 最大连接数
        manager.setDefaultMaxPerRoute(Max_PRE_ROUTE); // 路由最大连接数

        HttpHost httpHost = new HttpHost(host, port);
        manager.setMaxPerRoute(new HttpRoute(httpHost), MAX_ROUTE);

        //请求失败时,进行请求重试
        HttpRequestRetryHandler handler = new HttpRequestRetryHandler() {
            @Override
            public boolean retryRequest(IOException e, int i, HttpContext httpContext) {
                if (i > 3){
                    //重试超过3次,放弃请求
                    logger.error("重试超过3次,放弃请求");
                    return false;
                }
                if (e instanceof NoHttpResponseException){
                    //服务器没有响应,可能是服务器断开了连接,应该重试
                    logger.error("服务器没有响应,可能是服务器断开了连接,请重试");
                    return true;
                }
                if (e instanceof SSLHandshakeException){
                    // SSL握手异常
                    logger.error("SSL握手异常");
                    return false;
                }
                if (e instanceof InterruptedIOException){
                    //超时
                    logger.error("InterruptedIOException");
                    return false;
                }
                if (e instanceof UnknownHostException){
                    // 服务器不可达
                    logger.error("服务器不可达");
                    return false;
                }
                if (e instanceof ConnectTimeoutException){
                    // 连接超时
                    logger.error("连接超时");
                    return false;
                }
                if (e instanceof SSLException){
                    logger.error("SSLException");
                    return false;
                }

                HttpClientContext context = HttpClientContext.adapt(httpContext);
                HttpRequest request = context.getRequest();
                if (!(request instanceof HttpEntityEnclosingRequest)){
                    //如果请求不是关闭连接的请求
                    return true;
                }
                return false;
            }
        };

        CloseableHttpClient client = HttpClients.custom().setConnectionManager(manager).setRetryHandler(handler).build();
        return client;
    }

    /**
     * 设置post请求的参数
     * @param httpPost
     * @param params
     */
    private static void setPostParams(HttpPost httpPost, Map<String, Object> params){
        /*List<NameValuePair> nvps = new ArrayList<NameValuePair>();
        Set<String> keys = params.keySet();
        for (String key: keys){
            nvps.add(new BasicNameValuePair(key, params.get(key).toString()));
        }*/
    	StringEntity stringEntity;
		stringEntity = new StringEntity(JSON.toJSONString(params),HTTP.UTF_8);
		httpPost.setEntity(stringEntity);
        //HTTP.DEF_CONTENT_CHARSET
    }

    public static JSONObject post(String url, Map<String, Object> params){
    	logger.info("================执行HTTP请求开始====================");
    	HttpPost httpPost = new HttpPost(url);
        httpPost.addHeader("Content-Type", "application/json");
        setRequestConfig(httpPost);
        setPostParams(httpPost, params);
        CloseableHttpResponse response = null;
        InputStream in = null;
        JSONObject object = null;
        long startTime=System.currentTimeMillis();
        try {
            response = getHttpClient(url).execute(httpPost, HttpClientContext.create());
            
            long endTime=System.currentTimeMillis();
            
            
            startTime=System.currentTimeMillis();
            HttpEntity entity = response.getEntity();
            if (entity != null) {
                in = entity.getContent();
                String result = IOUtils.toString(in, "utf-8");
                Gson gson = new Gson();
//                object = gson.fromJson(result, JsonObject.class);
                object = JSONObject.parseObject(result);
            }
            
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            try{
                if (in != null) in.close();
                if (response != null) response.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        return object;
    }

    /**
     * 关闭连接池
     */
    public static void closeConnectionPool(){
        try {
            httpClient.close();
            manager.close();
            monitorExecutor.shutdown();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}

效果和注意

经过再次压测调试已发现由之前的4000+ms降低到了2000+ms，效果明细(虽然还是很慢)但是当我直接压测外部依赖的接口时发现这个接口同并发下居然也要2000+（真坑啊~~）。与方法内部调用损耗的时间只差了200ms。因此后续需要把依赖服务经行集群化、同时使用nginx做反向代理来提升依赖方法的响应时间。(注意：在设置连接池的路由和链接数属性时要对系统的调用量有个大概的估算，否则超出的请求连接就是直接抛弃无法请求了)