HttpClient 4.5 Best Practice

前言

最近在封装http请求相关的library，希望接口的行为和实现分离，这样底层library再切换不同http请求实现 例如：Apache HttpClient、OkHttp、AsyncHttpClient 等不影响上层调用行为。

关于 Apache HttpClient、OkHttp、AsyncHttpClient 我将逐一进行介绍，本文主要对Apache HttpClient做介绍。

Tutorial

HttpClient 4.5 Tutorial主要分为：基础（Fundamentals）、连接管理（Connection management）、http状态管理（HTTP state management）、http授权（HTTP authentication）、流式API（Fluent API）、http缓存（HTTP Caching）、高级主题（Advanced topics ）等7个专题。

个人建议使用HttpClient 4.5之前，需对上述7个专题做详细了解，以便在开发中更好的使用&调优。

介绍

HttpClient 是Apache官方推出的一款优秀的Http请求客户端library。

HttpClient 特点

• Client-side HTTP transport library based on HttpCore
• Based on classic (blocking) I/O
• Content agnostic

最佳实践

maven依赖

<dependency>
   <groupId>org.apache.httpcomponents</groupId>
    <artifactId>httpclient</artifactId>
    <version>4.5.2</version>
</dependency>

ApacheHttpClientImpl 类

package com.bytebeats.toolkit.http.impl;

import com.bytebeats.toolkit.annotation.ThreadSafe;
import com.bytebeats.toolkit.http.HttpRequestException;
import com.bytebeats.toolkit.http.config.FormParam;
import com.bytebeats.toolkit.http.config.HttpRequestConfig;
import org.apache.http.*;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

/**
 * https://hc.apache.org/httpcomponents-client-4.5.x/quickstart.html
 * https://hc.apache.org/httpcomponents-client-4.5.x/tutorial/html/connmgmt.html
 *
 * @author Ricky Fung
 * @create 2016-08-20 18:00
 */
@ThreadSafe
public class ApacheHttpClientImpl extends HttpClient {

    private final Logger logger = LoggerFactory.getLogger(ApacheHttpClientImpl.class);

    private final ThreadLocal<HttpClientContext> contextThreadLocal = new ThreadLocal(){
        @Override
        protected HttpClientContext initialValue() {

            return HttpClientContext.create();
        }
    };

    private final CloseableHttpClient httpclient;

    public ApacheHttpClientImpl(HttpRequestConfig config) {
        super(config);
        //multithreaded request execution
        PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
        // Increase max total connection to 200
        cm.setMaxTotal(this.config.getMaxConnections());
        // Increase default max connection per route to 50
        cm.setDefaultMaxPerRoute(this.config.getMaxConnectionPerHost());
        httpclient = HttpClients
                .custom()
                .setConnectionManager(cm)
                //.setRedirectStrategy(redirectStrategy)
                .setRetryHandler(new DefaultHttpRequestRetryHandler(0, false))
                .build();
    }

    @Override
    public String get(String url) throws HttpRequestException {

        try {
            HttpGet httpget = createHttpGet(url);
            return execute(httpget, contextThreadLocal.get(), new HttpResponseProcessor<String>() {
                @Override
                public String process(HttpResponse response) throws IOException {
                    if (response.getEntity() != null) {
                        return EntityUtils.toString(response.getEntity());
                    }
                    return null;
                }
            });
        } catch (Exception e) {
            logger.error("get error", e);
            throw new HttpRequestException("get error", e);
        }
    }

    @Override
    public byte[] getBytes(String url) throws HttpRequestException {
        try {
            HttpGet httpget = createHttpGet(url);
            execute(httpget, contextThreadLocal.get(), new HttpResponseProcessor<byte[]>() {
                @Override
                public byte[] process(HttpResponse response) throws IOException {
                    if (response.getEntity() != null) {
                        return EntityUtils.toByteArray(response.getEntity());
                    }
                    return null;
                }
            });

        } catch (Exception e) {
            logger.error("getAsBytes error", e);
            throw new HttpRequestException("getAsBytes error", e);
        }
        return null;
    }

    @Override
    public String postWithForm(String url, List<FormParam> params) throws HttpRequestException {
        try {
            HttpPost httppost = createHttpPost(url);
            List<NameValuePair> formparams = new ArrayList<NameValuePair>();
            for (FormParam param : params) {
                formparams.add(new BasicNameValuePair(param.getName(), param.getValue()));
            }
            UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, Consts.UTF_8);
            httppost.setEntity(entity);

            execute(httppost, contextThreadLocal.get(), new HttpResponseProcessor<String>() {
                @Override
                public String process(HttpResponse response) throws IOException {
                    if (response.getEntity() != null) {
                        return EntityUtils.toString(response.getEntity());
                    }
                    return null;
                }
            });
        } catch (Exception e) {
            logger.error("post error", e);
            throw new HttpRequestException("post error", e);
        }
        return null;
    }

    @Override
    public byte[] postWithFormBytes(String url, List<FormParam> params) throws HttpRequestException {
        return null;
    }

    @Override
    public String postWithBody(String url, String data) throws HttpRequestException {
        try {
            HttpPost httpPost = createHttpPost(url);
            StringEntity stringEntity = new StringEntity(data,
                    ContentType.create("plain/text", Consts.UTF_8));

            httpPost.setEntity(stringEntity);
            httpPost.setHeader("Content-type", "application/json");

            execute(httpPost, contextThreadLocal.get(), new HttpResponseProcessor<String>() {
                @Override
                public String process(HttpResponse response) throws IOException {
                    if (response.getEntity() != null) {
                        return EntityUtils.toString(response.getEntity());
                    }
                    return null;
                }
            });
        } catch (Exception e) {
            logger.error("post error", e);
            throw new HttpRequestException("post error", e);
        }
        return null;
    }

    @Override
    public byte[] postWithBodyBytes(String url, String data) throws HttpRequestException {
        try {
            HttpPost httpPost = createHttpPost(url);
            StringEntity stringEntity = new StringEntity(data, "UTF-8");
            stringEntity.setContentType("text/json");

            httpPost.setEntity(stringEntity);
            httpPost.setHeader("Content-type", "application/json");

            execute(httpPost, contextThreadLocal.get(), new HttpResponseProcessor<byte[]>() {
                @Override
                public byte[] process(HttpResponse response) throws IOException {
                    if (response.getEntity() != null) {
                        return EntityUtils.toByteArray(response.getEntity());
                    }
                    return null;
                }
            });
        } catch (Exception e) {
            logger.error("post error", e);
            throw new HttpRequestException("post error", e);
        }
        return null;
    }

    public <T> T execute(HttpUriRequest request, HttpResponseProcessor<T> processor) throws HttpRequestException {

        return this.execute(request, null, processor);
    }

    public <T> T execute(HttpUriRequest request, HttpContext context, HttpResponseProcessor<T> processor) throws HttpRequestException {

        CloseableHttpResponse response = null;
        try {
            if(context!=null){
                response = httpclient.execute(request, context);
            }else{
                response = httpclient.execute(request);
            }
            StatusLine statusLine = response.getStatusLine();
            if (statusLine.getStatusCode() == HttpStatus.SC_OK) {
                return processor.process(response);
            }
            logger.error("server:{} response code:{}", request.getURI(), statusLine.getStatusCode());
            throw new HttpRequestException(String.format("server:%s response error, statusCode:%s, reasonPhrase:%s",
                    request.getURI(), statusLine.getStatusCode(), statusLine.getReasonPhrase()));
        } catch (ClientProtocolException e) {
            // Handle protocol errors
            throw new HttpRequestException("doService protocol errors", e);
        } catch (IOException e) {
            // Handle I/O errors
            throw new HttpRequestException("doService IO errors", e);
        }finally {
            if (response != null) {
                try {
                    response.close();
                } catch (IOException e) {
                    logger.error("close response error", e);
                }
            }
        }
    }

    private HttpGet createHttpGet(String url) {

        HttpGet httpget = new HttpGet(url);
        httpget.setConfig(getRequestConfig(this.config));

        httpget.setHeader("Accept-Language", "zh-CN,zh;q=0.8,en;q=0.6");
        httpget.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36");

        return httpget;
    }

    private HttpPost createHttpPost(String url) {

        HttpPost httppost = new HttpPost(url);
        httppost.setConfig(getRequestConfig(this.config));

        httppost.setHeader("Accept-Language", "zh-CN,zh;q=0.8,en;q=0.6");
        httppost.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36");

        return httppost;
    }

    private RequestConfig getRequestConfig(HttpRequestConfig requestConfig){
        RequestConfig config = RequestConfig.custom()
                .setSocketTimeout(requestConfig.getSocketTimeout())
                .setConnectTimeout(requestConfig.getConnectTimeout())
                .setConnectionRequestTimeout(requestConfig.getConnectionRequestTimeout())
                .build();
        return config;
    }

    public static interface HttpResponseProcessor<T> {

        T process(HttpResponse entity) throws IOException;
    }

}

HttpRequestConfig 类

package com.bytebeats.toolkit.http.config;

/**
 * ${DESCRIPTION}
 *
 * @author Ricky Fung
 * @create 2016-08-24 19:06
 */
public class HttpRequestConfig {

    public static final HttpRequestConfig DEFAULT = new HttpRequestConfig();

    private int readTimeout = 6000;
    private int socketTimeout = 6000;
    private int connectTimeout = 6000;
    private int connectionRequestTimeout = 6000;

    private int maxConnections = 200;
    private int maxConnectionPerHost = 50;

    private boolean requestRetryEnabled;
    private boolean retryCount;

    //代理服务器
    private HttpHost proxy;

    public int getReadTimeout() {
        return readTimeout;
    }

    public void setReadTimeout(int readTimeout) {
        this.readTimeout = readTimeout;
    }

    public int getSocketTimeout() {
        return socketTimeout;
    }

    public void setSocketTimeout(int socketTimeout) {
        this.socketTimeout = socketTimeout;
    }

    public int getConnectTimeout() {
        return connectTimeout;
    }

    public void setConnectTimeout(int connectTimeout) {
        this.connectTimeout = connectTimeout;
    }

    public int getConnectionRequestTimeout() {
        return connectionRequestTimeout;
    }

    public void setConnectionRequestTimeout(int connectionRequestTimeout) {
        this.connectionRequestTimeout = connectionRequestTimeout;
    }

    public int getMaxConnectionPerHost() {
        return maxConnectionPerHost;
    }

    public void setMaxConnectionPerHost(int maxConnectionPerHost) {
        this.maxConnectionPerHost = maxConnectionPerHost;
    }

    public boolean isRequestRetryEnabled() {
        return requestRetryEnabled;
    }

    public void setRequestRetryEnabled(boolean requestRetryEnabled) {
        this.requestRetryEnabled = requestRetryEnabled;
    }

    public boolean isRetryCount() {
        return retryCount;
    }

    public void setRetryCount(boolean retryCount) {
        this.retryCount = retryCount;
    }

    public int getMaxConnections() {
        return maxConnections;
    }

    public void setMaxConnections(int maxConnections) {
        this.maxConnections = maxConnections;
    }

    public HttpHost getProxy() {
        return proxy;
    }

    public void setProxy(HttpHost proxy) {
        this.proxy = proxy;
    }
}

FormParam 类

package com.bytebeats.toolkit.http.config;

/**
 * ${DESCRIPTION}
 *
 * @author Ricky Fung
 * @create 2016-08-24 16:17
 */
public class FormParam {
    private final String name;
    private final String value;

    public FormParam(String name, String value) {
        this.name = name;
        this.value = value;
    }

    public String getName() {
        return this.name;
    }

    public String getValue() {
        return this.value;
    }
}

参考资料

1. Fundamentals

2. Connection management

3. HTTP state management

4. HTTP authentication

5. Fluent API

6. HTTP Caching

7. Advanced topics

参考资料

HttpClient 4.5 Tutorial：https://hc.apache.org/httpcomponents-client-4.5.x/tutorial/html/index.html