HttpClient使用总结

依赖

<dependency>
    <groupId>org.apache.httpcomponents</groupId>
    <artifactId>httpmime</artifactId>
    <version>4.5.3</version>
</dependency>
<!--GetByteEncode-->
<dependency>
    <groupId>com.googlecode.juniversalchardet</groupId>
    <artifactId>juniversalchardet</artifactId>
    <version>1.0.3</version>
</dependency>
<!-- commons-io -->
<dependency>
    <groupId>commons-io</groupId>
    <artifactId>commons-io</artifactId>
    <version>2.5</version>
</dependency>

使用工具

package com.isi.util;

import org.apache.commons.io.IOUtils;
import org.apache.http.*;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.ssl.SSLContexts;
import org.apache.http.util.EntityUtils;
import org.mozilla.universalchardet.UniversalDetector;

import javax.net.ssl.SSLContext;
import java.io.*;
import java.net.URI;
import java.security.KeyManagementException;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class DownloadUtils {
    private static final String ua = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36";
    private static int default_timeout = 500*1000;
    private static PoolingHttpClientConnectionManager cm = null;
    private static SSLConnectionSocketFactory sslsf1;
    private static Map<String, String> defaultHeader = new HashMap<>();
    static {
        sslsf1 = formSslsf1();
        Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
                .register("https", sslsf1)
                .register("http", new PlainConnectionSocketFactory())
                .build();

        cm = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
        cm.setMaxTotal(200);
        cm.setDefaultMaxPerRoute(20);


        defaultHeader.put("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8");
        defaultHeader.put("Accept-Encoding", "gzip, deflate");
        defaultHeader.put("accept-language", "zh-CN,zh;q=0.9,en;q=0.8");
        defaultHeader.put("cache-control", "max-age=0");
        defaultHeader.put("upgrade-insecure-requests", "1");
        defaultHeader.put("user-agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36");
        defaultHeader.put("referer", "https://www.baidu.com");
    }

    /**
     * //unable to find valid certification path to requested target 异常解决
     *
     * @return
     */
    private static SSLConnectionSocketFactory formSslsf1() {
        SSLConnectionSocketFactory sslConnectionSocketFactory = null;
        SSLContext sslContext = null;
        try {
            sslContext = SSLContexts.custom()
                    .loadTrustMaterial(null, new org.apache.http.ssl.TrustStrategy() {
                        @Override
                        public boolean isTrusted(java.security.cert.X509Certificate[] x509Certificates, String s) throws CertificateException {
                            return true;
                        }
                    })
                    .build();
        } catch (NoSuchAlgorithmException e) {
            e.printStackTrace();
        } catch (KeyManagementException e) {
            e.printStackTrace();
        } catch (KeyStoreException e) {
            e.printStackTrace();
        }
        sslConnectionSocketFactory = new SSLConnectionSocketFactory(
                sslContext,
                NoopHostnameVerifier.INSTANCE);
        return sslConnectionSocketFactory;

    }

    /**
     * 构造一个单例的httpclient
     */
    static class Sington {
        private static CloseableHttpClient httpClient = HttpClients.custom()
                .setConnectionManager(cm)
                .setDefaultRequestConfig(RequestConfig.custom()
                        .setSocketTimeout(default_timeout)
                        .setConnectionRequestTimeout(default_timeout)
                        .setConnectTimeout(default_timeout).build()).build();
    }


    /**
     * get参数汇总
     *
     * @param url
     * @param charset
     * @param headers
     * @param ip
     * @param port
     * @param timeout
     * @return
     */
    private static String totalGet(String url, String charset, Map<String, String> headers, String ip, Integer port, Integer timeout) {
        String result = null;
        HttpGet httpGet = new HttpGet();
        CloseableHttpClient httpClient = Sington.httpClient;
        initconfig(httpGet, timeout, ip, port);
        initHeader(httpGet, headers);
        CloseableHttpResponse response = null;
        try {
//            response = httpClient.execute(httpGet);
            httpGet.setURI(new URI(url));
            HttpClientContext context = HttpClientContext.create();
            response = httpClient.execute(httpGet,context);

            //获取跳转url
            List<URI> redirectURIs = context.getRedirectLocations();
            String redirectUrl = null;
            if (redirectURIs != null && !redirectURIs.isEmpty()) {
                URI finalURI = redirectURIs.get(redirectURIs.size() - 1);
                redirectUrl = finalURI.toString();
                System.out.println("finalURI=" + redirectUrl);
            }

            int statusCode = response.getStatusLine().getStatusCode();
            if (statusCode == HttpStatus.SC_OK) {
                result = getResponseContent(charset, response);
            }
        } catch (Exception e) {
           e.printStackTrace();
        } finally {
            httpGet.releaseConnection();
            closeResponse(response);
        }
        return result;
    }

    /**
     * post参数汇总
     *
     * @param url
     * @param charset
     * @param headers
     * @param dataStr
     * @param ip
     * @param port
     * @param timeout
     * @return
     */
    public static String totalPost(String url, String charset, Map<String, String> headers, String dataStr, String ip, Integer port, Integer timeout) {
        String result = null;
        if (charset == null) {
            charset = "utf-8";
        }
        HttpPost httpPost = new HttpPost(url);
        CloseableHttpClient httpClient = Sington.httpClient;
        initconfig(httpPost, timeout, ip, port);
        initHeader(httpPost, headers);
        initPostEntity(dataStr, httpPost, charset);
        CloseableHttpResponse response = null;
        try {
            response = httpClient.execute(httpPost);
            int statusCode = response.getStatusLine().getStatusCode();
            System.out.println("statusCode====="+statusCode);
            if (statusCode == HttpStatus.SC_OK) {
                result = EntityUtils.toString(response.getEntity(), charset);
            }
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            httpPost.releaseConnection();
            closeResponse(response);
        }
        return result;
    }

    /**
     * post参数汇总
     *
     * @param url
     * @param charset
     * @param headers
     * @param datas
     * @param ip
     * @param port
     * @param timeout
     * @return
     */
    public static String totalPost(String url, String charset, Map<String, String> headers, Map<String, Object> datas, String ip, Integer port, Integer timeout) {
        System.out.println("start post url====>>>"+url);
        long t1 = System.currentTimeMillis();
        String result = null;
        if (charset == null) {
            charset = "utf-8";
        }
        HttpPost httpPost = new HttpPost(url);
        CloseableHttpClient httpClient = Sington.httpClient;
        initconfig(httpPost, timeout, ip, port);
        initHeader(httpPost, headers);
        initPostEntity(datas, httpPost,charset);
        CloseableHttpResponse response = null;
        try {
            response = httpClient.execute(httpPost);
            int statusCode = response.getStatusLine().getStatusCode();
            System.out.println("statusCode====="+statusCode);
            if (statusCode == HttpStatus.SC_OK) {
                result = EntityUtils.toString(response.getEntity(), charset);
            }
            long t2 = System.currentTimeMillis();
            System.out.println("request cost=====>>>>>"+(t2-t1)+" ms");
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            httpPost.releaseConnection();
            closeResponse(response);
        }
        return result;
    }


    public static boolean down2File(String url, String ip, Integer port, final String destPath, boolean ddxc) {
        // 创建默认的httpClient实例
        CloseableHttpClient httpClient = Sington.httpClient;
        CloseableHttpResponse httpResponse = null;
        RequestConfig requestConfig = getRequestConfig(1000 * 30, ip, port);
        RandomAccessFile raf = null;
        InputStream in = null;
        File tmpFile = null;
        File dFile = null;

        // 用get方法发送http请求
        HttpGet httpGet = new HttpGet(url);
        httpGet.setConfig(requestConfig);
        httpGet.addHeader("User-Agent", ua);
        dFile = new File(destPath);
        if (dFile.exists()) {
            System.out.println("file already exist skip ");
            return true;
        }
        tmpFile = new File(destPath + "_tmp");
        long downloadSize = 0L;
        long startIndex = 0L;

        if (ddxc) {
            if (tmpFile.exists() && tmpFile.isFile()) {
                downloadSize = tmpFile.length();
                startIndex = downloadSize;
            }
            httpGet.addHeader("Range", "bytes=" + startIndex + "-");//请求断点
        } else {
            if (tmpFile.exists() && tmpFile.isFile()) {
                boolean deleteFlag = tmpFile.delete();
                System.out.println("delete flag deleteFlag=="+ deleteFlag);
            }
        }

        try {

            // response实体
            httpResponse = httpClient.execute(httpGet);
            final HttpEntity entity = httpResponse.getEntity();

            if (null != entity) {
                int statusCode = httpResponse.getStatusLine().getStatusCode();
                System.out.println("statusCode is "+statusCode);
                if (statusCode == 200 || statusCode == 206) {
                    raf = new RandomAccessFile(tmpFile, "rwd");
                    raf.seek(startIndex);
                    in = entity.getContent();
                    byte[] buffer = new byte[1024 * 10];
                    int size = 0;
                    System.out.println("start download ");
                    while ((size = in.read(buffer)) != -1) {
                        raf.write(buffer, 0, size);

                    }
                    System.out.println("download over!!!!");
                }

            }
        } catch (Exception e) {
            System.out.println("httpclient请求失败 url===="+url+":::::::"+ e.getMessage());
            return false;
        } finally {
            httpGet.releaseConnection();
            //规范流的关闭操作，否则如果报异常，流是没有正常关闭的
            if (in != null) {
                try {
                    in.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }

            if (raf != null) {
                try {
                    raf.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }

            closeResponse(httpResponse);
        }
        return tmpFile.renameTo(dFile);//重命名之前，该关闭的流都要关闭
    }

    public static byte[]  down2Byte(String url, String ip, Integer port) {
        // 创建默认的httpClient实例
        CloseableHttpClient httpClient =Sington.httpClient;
        CloseableHttpResponse httpResponse = null;
        RequestConfig requestConfig = getRequestConfig(1000 * 30, ip, port);

        // 用get方法发送http请求
        HttpGet httpGet = new HttpGet(url);
        httpGet.setConfig(requestConfig);
        httpGet.addHeader("User-Agent", ua);
        try {
            // response实体
            httpResponse = httpClient.execute(httpGet);
            final HttpEntity entity = httpResponse.getEntity();

            if (null != entity) {
                int statusCode = httpResponse.getStatusLine().getStatusCode();
                System.out.println("statusCode is "+statusCode+" url=="+url);
                if (statusCode == 200 || statusCode == 206) {
                    byte[] contentBytes = IOUtils.toByteArray(httpResponse.getEntity().getContent());
                    return  contentBytes;
                }

            }
        } catch (Exception e) {
            System.out.println("httpclient请求失败 url===="+url+":::::::"+ e.getMessage());
            return null;
        } finally {
            //规范流的关闭操作，否则如果报异常，流是没有正常关闭的
            httpGet.releaseConnection();
            closeResponse(httpResponse);
        }
        return null;
    }



    /**
     * 初始化post值
     *
     * @param datas
     * @param httpPost
     */
    private static void initPostEntity(Map<String, Object> datas, HttpPost httpPost, String charset) {
        List<NameValuePair> params = new ArrayList<NameValuePair>();
        for (Map.Entry<String, Object> entry : datas.entrySet()) {
            String key = entry.getKey();
            Object value = entry.getValue();
            if(key==null || value==null){
                continue;
            }
            params.add(new BasicNameValuePair(key, value.toString()));
        }
        try {
            UrlEncodedFormEntity entity = new UrlEncodedFormEntity(params, charset);

            httpPost.setEntity(entity);
        } catch (UnsupportedEncodingException e) {
            e.printStackTrace();
        }
    }


    /**
     * 初始化post值
     *
     * @param dataStr
     * @param httpPost
     * @param charset
     */
    private static void initPostEntity(String dataStr, HttpPost httpPost, String charset) {
        StringEntity stringEntity = new StringEntity(dataStr, charset);
        httpPost.setEntity(stringEntity);
    }

    public static String getResponseContent(String charset, HttpResponse httpResponse) throws IOException {
        if (charset == null) {
            byte[] contentBytes = IOUtils.toByteArray(httpResponse.getEntity().getContent());
            String encoding = getEncoding(contentBytes);
            System.out.println("detect charset======"+encoding);
            return new String(contentBytes,encoding);
        } else {
            return IOUtils.toString(httpResponse.getEntity().getContent(), charset);
        }
    }


    /**
     * 关闭响应
     *
     * @param httpResponse
     */
    private static void closeResponse(CloseableHttpResponse httpResponse) {
        if (httpResponse != null) {
            try {
                EntityUtils.consume(httpResponse.getEntity());
                httpResponse.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }


    /**
     * 获取编码
     * @param bytes
     * @return
     */
    public static String getEncoding(byte[] bytes) {
        String defaultEncoding = "UTF-8";
        UniversalDetector detector = new UniversalDetector(null);
        detector.handleData(bytes, 0, bytes.length);
        detector.dataEnd();
        String encoding = detector.getDetectedCharset();
        detector.reset();
        System.out.println("字符编码是："+ encoding);
        if (encoding == null) {
            encoding = defaultEncoding;
        }
        return encoding;
    }


    /**
     * 初始化config
     *
     * @param request
     * @param headers
     */
    private static void initHeader(HttpUriRequest request, Map<String, String> headers) {
        if (headers != null) {
            for (Map.Entry<String, String> entry : headers.entrySet()) {
                String name = entry.getKey();
                String value = entry.getValue();
                request.addHeader(name, value);
            }
        } else {
            for (Map.Entry<String, String> entry : defaultHeader.entrySet()) {
                String name = entry.getKey();
                String value = entry.getValue();
                request.addHeader(name, value);
            }
        }
    }

    /**
     * 初始化 config
     *
     * @param httpGet
     * @param timeout
     * @param ip
     * @param port
     */
    private static void initconfig(HttpGet httpGet, Integer timeout, String ip, Integer port) {
        RequestConfig requestConfig = getRequestConfig(timeout, ip, port);
        httpGet.setConfig(requestConfig);
    }


    /**
     * 初始化config
     *
     * @param httpPost
     * @param timeout
     * @param ip
     * @param port
     */
    private static void initconfig(HttpPost httpPost, Integer timeout, String ip, Integer port) {
        RequestConfig requestConfig = getRequestConfig(timeout, ip, port);
        httpPost.setConfig(requestConfig);
    }

    /**
     * 构造requestconfig
     *
     * @param timeout
     * @param ip
     * @param port
     * @return
     */
    private static RequestConfig getRequestConfig(Integer timeout, String ip, Integer port) {
        RequestConfig.Builder confBuilder = RequestConfig.custom();
        if (timeout == null) {
            timeout = default_timeout;
        }
        confBuilder.setConnectTimeout(timeout).setSocketTimeout(timeout).setConnectionRequestTimeout(timeout);
        if (ip != null && port != null) {
            confBuilder.setProxy(new HttpHost(ip, port));
        }
        confBuilder.setRedirectsEnabled(true);
        return confBuilder.build();
    }

}