HTTP连接池

前言

最早接触HTTP连接池，是在HTTP传输数据过程中，或因网络节点问题，或因网络波动，或因Tomcat连接爆棚等等，导致数据传输失败。想通过HTTP链接池，优化性能，但是因为公司项目架构的问题，最后使用了RocketMQ。下面是早前写的Demo。

正文

HTTP连接池优势

1.复用http连接,省去了tcp的3次握手和4次挥手的时间,极大降低请求响应的时间

2.自动管理tcp连接,不用人为地释放/创建连接

注意点

不要多个应用创建连接池去对应一个应用。嗯，大抵没记错的话，会使被请求方连接数的不到释放，最后服务宕机。

代码正文

package com.taskorder.util;


import org.apache.commons.codec.Charsets;
import org.apache.http.*;
import org.apache.http.client.HttpClient;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.conn.ClientConnectionManager;
import org.apache.http.conn.ConnectionKeepAliveStrategy;
import org.apache.http.conn.params.ConnManagerParams;
import org.apache.http.conn.params.ConnPerRouteBean;
import org.apache.http.conn.routing.HttpRoute;
import org.apache.http.conn.scheme.PlainSocketFactory;
import org.apache.http.conn.scheme.Scheme;
import org.apache.http.conn.scheme.SchemeRegistry;
import org.apache.http.conn.ssl.SSLSocketFactory;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager;
import org.apache.http.message.BasicHeaderElementIterator;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.params.BasicHttpParams;
import org.apache.http.params.HttpParams;
import org.apache.http.protocol.ExecutionContext;
import org.apache.http.protocol.HTTP;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.nio.charset.Charset;
import java.util.*;
import java.util.Map.Entry;
import java.util.concurrent.TimeUnit;

/**
 * 邙邙
 * 连接池，get,post,
 * 2020.6.21
 */
public class HttpClientAgent {
    private static Logger logger = LoggerFactory.getLogger(HttpClientAgent.class);
    private static HttpParams params;
    private static ClientConnectionManager cm;
    private static HttpClient httpClient;
    private static int seeNumber =0;
    static{
        System.out.println("初始化HttpClientTest~~~开始");
        params=new BasicHttpParams();
        /**
        *设置最大连接数,最大连接数就是连接池允许的最大连接数,httpclient设置的最大连接数绝对不能超过tomcat设置的最大连接数，否则tomcat的连接就会被httpclient连接池一直占用，直到系统挂掉。
         * 一般上tomcat最大连接默认为200，排队等待为100
         */
        ConnManagerParams.setMaxTotalConnections(params,180);
        /**说明：设置每个路由最大连接数。这里要特别提到route（路由）最大连接数这个参数呢，
         因为这个参数的默认值为2，如果不设置这个参数值，默认情况下对于同一个目标机器的最大并发连接只有2个
         ，这意味着如果你正在执行一个针对某一台目标机器的抓取任务的时候，哪怕你设置连接池的最大连接数为300
         ，但是实际上还是只有2个连接在工作，其他剩余的298个连接都在等待，都是为别的目标机器服务的。
         */
        ConnPerRouteBean connPerRoute = new ConnPerRouteBean(100);
        /**
         * 设置www.xxx.com这个网站对应的路由连接最大为100，因为此工具只为一个URL提供服务，所以要把此网站的最大连接数设置为100，保持与路由最大连接数一致，不然只会取最小值
         * HttpHost(要访问的域名，要访问的端口);
         * */

        HttpHost localhost = new HttpHost("127.0.0.1", 10086);
        connPerRoute.setMaxForRoute(new HttpRoute(localhost),100);
        ConnManagerParams.setMaxConnectionsPerRoute(params,connPerRoute);
        // 设置访问协议模式
        SchemeRegistry schemeRegistry=new SchemeRegistry();
        schemeRegistry.register( new Scheme("http", PlainSocketFactory.getSocketFactory(),80));
        schemeRegistry.register(new Scheme("https", SSLSocketFactory.getSocketFactory(),443));
//        ClientConnectionManager cm=new ThreadSafeClientConnManager(params,schemeRegistry);
        cm=new ThreadSafeClientConnManager(params,schemeRegistry);
//        httpClient=new DefaultHttpClient(cm,params);
//        httpClient=
        DefaultHttpClient   defaultHttpClient  = new DefaultHttpClient(cm,params);
        //设置连接过期时间
        defaultHttpClient.setKeepAliveStrategy(new ConnectionKeepAliveStrategy(){

            public long getKeepAliveDuration(HttpResponse response, HttpContext context) {
                HeaderElementIterator it = new BasicHeaderElementIterator(response.headerIterator(HTTP.CONN_KEEP_ALIVE));
                while (it.hasNext()) {
                    HeaderElement he = it.nextElement();
                    String param = he.getName();
                    String value = he.getValue();
                    if (value != null && param.equalsIgnoreCase("timeout")) {
                        try {
                            return Long.parseLong(value) * 1000;
                        } catch(NumberFormatException ignore) {

                        }
                    }
                }
                HttpHost target = (HttpHost) context.getAttribute( ExecutionContext.HTTP_TARGET_HOST);
//                if ("www.xxx.com".equalsIgnoreCase(target.getHostName())) {
                if ("127.0.0.1".equalsIgnoreCase(target.getHostName())) {
                    // 对于xxx这个路由的连接，保持60秒
                    System.out.println("--------xxx---------");
                    return 60 * 1000;
                } else {
                    // 其他路由保持40秒
                    return 40 * 1000;
                }

            }


        });
        /**
         *   使用线程安全的连接管理来创建HttpClient
         */
        httpClient = defaultHttpClient;
        System.out.println("初始化HttpClientTest~~~结束");
    }
    // 获取Http的连接
//    public static HttpClient getHttpClient() {
//        return new DefaultHttpClient(cm, params);
//    }

    public static class IdleConnectionMonitorThread extends Thread{
        private final ClientConnectionManager connMgr;
        private volatile boolean shutdown;
        public IdleConnectionMonitorThread(ClientConnectionManager connMgr) {
            super(); this.connMgr = connMgr;
        }

        @Override
        public void run() {
            ++seeNumber;
            System.out.println("进入run()方法"+seeNumber);
            try {
                while (!shutdown) {
                    synchronized (this) {
                        wait(5000);
                        // 关闭过期的连接
                        connMgr.closeExpiredConnections();
                        // 关闭空闲时间超过30秒的连接
                        connMgr.closeIdleConnections(30, TimeUnit.SECONDS);
                        System.out.println("关闭方法"+seeNumber);
                        shutdown();
                    }
                }
            } catch (InterruptedException ex) {
                System.out.println("异常");
                ex.printStackTrace();
            }
        }

        public void shutdown() {
            shutdown = true;
            synchronized (this) {
                notifyAll();
            }
        }


    }



    public static String get(String url, Map<String, String> paramMapTwo,
                           List<Header> headers) throws Exception {

        /**
         * 1.连接的有效性检测是所有连接池都面临的一个通用问题，大部分HTTP服务器为了控制资源开销，并不会
         * 永久的维护一个长连接，而是一段时间就会关闭该连接。放回连接池的连接，如果在服务器端已经关闭，客
         * 户端是无法检测到这个状态变化而及时的关闭Socket的。这就造成了线程从连接池中获取的连接不一定是有效的。
         * 这个问题的一个解决方法就是在每次请求之前检查该连接是否已经存在了过长时间，可能已过期
         * 2.清除连接池中所有过期的连接和关闭一定时间空闲的连接
         * */
        new IdleConnectionMonitorThread(cm).start();

        // 根据URL创建HttpGet实例
        HttpGet httpGet = new HttpGet(url);
        System.out.println("httpGet"+httpGet);
//        HttpGet httpGet = new HttpGet(uriBuilder.build());
        if (headers != null) {
            // 添加请求首部
            for (Header header : headers) {
                httpGet.addHeader(header);
            }
        }
        String data = null;
        HttpResponse response = null;
        try{
            // 执行get请求，得到返回体
            response = httpClient.execute(httpGet);
            System.out.println("response"+response);
            // 判断是否正常返回

            if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
                // 解析数据
                data = EntityUtils.toString(response.getEntity(), Charsets.UTF_8);
                System.out.println(data);
            }

        }catch(IOException e){
            e.printStackTrace();
        }finally{
            if (response != null){
                EntityUtils.consumeQuietly(response.getEntity());
            }
        }


        return data;
    }

    public static String get(String url, Map<String, String> paramMapTwo)
            throws Exception {

        return get(url, paramMapTwo, null);
    }


    public static String httpGet(String url) {
        HttpGet httpGet = new HttpGet(url);
        CloseableHttpResponse response = null;
//        HttpClient client =getHttpClient();
        try {
            response = (CloseableHttpResponse) httpClient.execute(httpGet);
            String result = EntityUtils.toString(response.getEntity());
            int code = response.getStatusLine().getStatusCode();
            if (code == HttpStatus.SC_OK) {
                return result;
            } else {
                logger.error("请求{}返回错误码：{},{}", url, code,result);
                return null;
            }
        } catch (IOException e) {
            logger.error("http请求异常，{}",url,e);
        } finally {
            try {
                if (response != null)
                    response.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        return null;
    }

    public static String httpPost(String url,Map<String, String> paramsTwo) {

        String result= "";
        new IdleConnectionMonitorThread(cm).start();
//        HttpClient client =new DefaultHttpClient();
        //POST的URL
        HttpPost httppost=new HttpPost(url);
        //建立HttpPost对象
//        List<NameValuePair> params=new ArrayList<NameValuePair>();
        //建立一个NameValuePair数组，用于存储欲传送的参数
//        params.add(new BasicNameValuePair("pwd","2544"));
        List<NameValuePair> nvps = new ArrayList <NameValuePair>();

        Set<String> keySet = paramsTwo.keySet();
        for(String key : keySet) {
            nvps.add(new BasicNameValuePair(key, paramsTwo.get(key)));
        }
        //添加参数
        try {
            httppost.setEntity(new UrlEncodedFormEntity(nvps,HTTP.UTF_8));
        }catch(UnsupportedEncodingException e) {
            e.printStackTrace();
        }

        try {
            //设置编码
            HttpResponse response=httpClient.execute(httppost);
//            HttpResponse response=client.execute(httppost);
            //发送Post,并返回一个HttpResponse对象
            //Header header = response.getFirstHeader("Content-Length");
            //String Length=header.getValue();
            // 上面两行可以得到指定的Header
            /**
             * 得到http响应结果的状态码为200则获取返回值
             */
            if(response.getStatusLine().getStatusCode()==200){//如果状态码为200,就是正常返回
                result=EntityUtils.toString(response.getEntity(),Charsets.UTF_8);
                //得到返回的字符串
                System.out.println(result);
                //打印输出
                //如果是下载文件,可以用response.getEntity().getContent()返回InputStream
            }else{
                System.out.println("result=空");
            }

        }catch(IOException e) {
            e.printStackTrace();
        }


        return result;
    }
    //测试
    public static void main(String[] args) throws Exception {
        Map<String, String> paramMap = new HashMap<String, String>();
        paramMap.put("id","1");

        long start=System.currentTimeMillis();

        for (int i = 0; i < 500; i++) {

            get("https://www.自己随便填个网址.", paramMap);
		}
        System.err.println("---------------------------postTest---------------------------");
        System.err.println(System.currentTimeMillis()-start);

    }

}