HttpClients 工具类

HttpUtils.java

package com.test.http.util;

import java.io.IOException;
import java.net.URI;
import java.nio.charset.Charset;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.RedirectStrategy;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.conn.ssl.TrustStrategy;
import org.apache.http.entity.ContentType;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.client.LaxRedirectStrategy;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.protocol.HttpCoreContext;
import org.apache.http.ssl.SSLContextBuilder;
import org.apache.http.util.EntityUtils;
import org.apache.log4j.Logger;

import com.test.http.HttpResult;
import com.test.http.util.StringUtils;

/**
 * http工具类
 * http请求头大全：http://tools.jb51.net/table/http_header
 * https解决方案：http://www.cnblogs.com/yangmengdx3/p/6322380.html
 * @author youxiang
 *
 */
public class HttpUtils {

    private static Logger log = Logger.getLogger(HttpUtils.class);

    /**
     * User-Agent 浏览器用户标识
     */
    public static final String USER_AGENT_VALUE="Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.108 Safari/537.36";

    /**
     * http请求参数编码
     */
    public static final String HTTP_PARAMS_CODING="UTF-8";

    private static final String HTTP = "http";
    private static final String HTTPS = "https";
    private static SSLConnectionSocketFactory sslFactory = null;
    private static PoolingHttpClientConnectionManager connectionManager = null;
    private static SSLContextBuilder builder = null;
    private static RequestConfig requestConfig = null;
    private static RedirectStrategy redirectStrategy = null;

    static {
        try {
            builder = new SSLContextBuilder();
            // HTTPS全部信任 不做身份鉴定
            builder.loadTrustMaterial(null, new TrustStrategy() {
                @Override
                public boolean isTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {
                    return true;
                }
            });
            sslFactory = new SSLConnectionSocketFactory(builder.build(), new String[]{"SSLv2Hello", "SSLv3", "TLSv1", "TLSv1.2"}, null, NoopHostnameVerifier.INSTANCE);
            Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create().register(HTTP, new PlainConnectionSocketFactory()).register(HTTPS, sslFactory).build();
            connectionManager = new PoolingHttpClientConnectionManager(registry);
            connectionManager.setMaxTotal(300);//连接池最大并发连接数
            connectionManager.setDefaultMaxPerRoute(100);//单路由的最大并发连接数
            /**
             * 设置请求和.设置从connect Manager获取Connection 超时时间.传输超时时间
             * .setConnectTimeout(5000) 建立连接的时间
             * .setConnectionRequestTimeout(5000) 从连接池中获取连接的时间
             * .setSocketTimeout(10000)  数据传输处理时间
             */
            requestConfig = RequestConfig.custom().setConnectTimeout(20000).build();
            //解决post重定向问题
            redirectStrategy = new LaxRedirectStrategy();
        } catch (Exception e) {
            log.error("设置httpclient参数失败", e);
        }
    }

    private HttpUtils() {
        throw new IllegalStateException("Utility class");
    }

    /**
     * 获取HttpClient对象
     * @return
     */
    public static HttpClient getHttpClient() {
        return HttpClients.custom().setRedirectStrategy(redirectStrategy).setSSLSocketFactory(sslFactory).setConnectionManager(connectionManager).setDefaultRequestConfig(requestConfig).setConnectionManagerShared(true).build();
    }

    /**
     * 获取网页内容
     * @param response
     * @return
     * @throws IOException 
     * @throws Exception
     */
    public static HttpResult getHtml(HttpResponse response) throws IOException {
        HttpResult result = new HttpResult();
        String html = "";
        String charsetStr = HttpUtils.HTTP_PARAMS_CODING;
        HttpEntity entity = response.getEntity();
        if(entity != null){
            byte [] bytes = EntityUtils.toByteArray(entity);
            if(bytes!=null && bytes.length>0){
                Charset charset = null;
                // 如果头部Content-Type中包含了编码信息，那么我们可以直接在此处获取
                ContentType contentType = ContentType.getOrDefault(entity);
                charset = contentType.getCharset();
                //charSet = EntityUtils.getContentCharSet(entity);//方法已过时
                // 如果头部中没有，那么我们需要 查看页面源码，这个方法虽然不能说完全正确，因为有些粗糙的网页编码者没有在页面中写头部编码信息
                if (charset==null) {
                    try{
                        String regEx = "(?=<meta).*?(?<=charset=[\\'|\\\"]?)([a-zA-Z0-9-]*)";
                        //获取页面中header部分的编码
                        String charsetName = HttpUtils.getStrings(new String(bytes), regEx, 2, 1);
                        charset = Charset.forName(charsetName);
                    }catch(Exception e){
                        log.error("未找到编码 error："+e.getMessage());
                    }
                }
                if(charset!=null) {
                    charsetStr = charset.name();
                }
                log.debug("charSet:"+charsetStr);
                // 至此，我们可以将原byte数组按照正常编码专成字符串输出（如果找到了编码的话）
                html = new String(bytes, charsetStr);
                log.debug(html);
            }
        }
        result.setResponseBody(html);
        result.setCharSet(charsetStr);
        return result;
    }

    /**
     * 获取响应信息
     * @param httpContext
     * @return
     * @throws IOException 
     */
    public static HttpResult getHttpInfo(HttpClientContext httpContext) throws IOException {
        HttpResult result = new HttpResult();
        HttpUriRequest request = (HttpUriRequest) httpContext.getAttribute(HttpCoreContext.HTTP_REQUEST);
        //获取请求链接
        StringBuilder  requestUrl = new StringBuilder();
        List<URI> uriList = httpContext.getRedirectLocations();
        if(uriList==null) {
            HttpHost currentHost = (HttpHost) httpContext.getAttribute(HttpCoreContext.HTTP_TARGET_HOST);
            requestUrl.append((request.getURI().isAbsolute()) ? request.getURI().toString() : (currentHost.toURI() + request.getURI()));
        }else {
            for(URI uri:uriList) {
                requestUrl.append(uri.toString()).append("\r\n");
            }
        }
        result.setRequestUrl(requestUrl.toString());
        //获取请求头
        StringBuilder  requestHeaderStr = new StringBuilder ();
        Header[] requestHeaders = request.getAllHeaders();
        if(requestHeaders!=null) {
            for(Header requestHeader:requestHeaders) {
                requestHeaderStr.append(requestHeader.toString()).append("\r\n");
            }
        }
        result.setRequestHeader(requestHeaderStr.toString());
        //获取响应头
        HttpResponse response = (HttpResponse) httpContext.getAttribute(HttpCoreContext.HTTP_RESPONSE);
        StringBuilder  responseHeaderStr = new StringBuilder ();
        Header[] responseHeaders = response.getAllHeaders();
        if(responseHeaders!=null) {
            for(Header responseHeader:responseHeaders) {
                responseHeaderStr.append(responseHeader.toString()).append("\r\n");
            }
        }
        result.setResponseHeader(responseHeaderStr.toString());
        //获取网页内容
        HttpResult htmlResult = HttpUtils.getHtml(response);
        result.setResponseBody(htmlResult.getResponseBody());
        result.setCharSet(htmlResult.getCharSet());
        int statusCode = response.getStatusLine().getStatusCode();
        result.setStatusCode(statusCode);
        if(statusCode>=400){
            //输入信息
            log.info(result.toString());
            throw new CustomException(requestUrl+" Error:请求失败，返回码->"+statusCode);
        }
        return result;
    }

    /**
     * 根据正则表达式获取文本
     * @param html 文本
     * @param regex 正则表达式
     * @param template 模板 （指定第几个模板）从1开始
     * @param match_no 匹配数字（获取第几个） 0表示随机选取，1表示选取第一个
     */
    public static String getStrings(String html, String regex, int template, int matchNo) {
        String result="";
        try{
            Pattern p = Pattern.compile(regex, Pattern.DOTALL | Pattern.CASE_INSENSITIVE);
            Matcher m = p.matcher(html);
            ArrayList<String> strs = new ArrayList<>();
            while (m.find()) {
                strs.add(m.group(template-1));
            }
            if(matchNo==0){
                int index = new Random().nextInt(strs.size());
                result=strs.get(index);
            }else if(matchNo>0){
                result=strs.get(matchNo-1);
            }
        }catch(Exception e){
            throw new CustomException("找不到值!正则表达式:"+regex+";模板:"+template+";匹配数字:"+matchNo+";异常:"+e.getMessage());
        }
        return result;
    }

    /**
     * 通过前后边界获取字符
     * @param html 文本
     * @param LB 左边开始字符
     * @param RB 右边结束字符
     */
    public static String getStrings(String html, String leftBefore, String rightBefore){
        String result="";
        try{
            int index = 0;
            if(StringUtils.isNotEmpty(leftBefore)) {
                index = html.indexOf(leftBefore);
                if(index<0) {
                    throw new CustomException("返回数据找不到LB值!LB:"+leftBefore);
                }
                index = index + leftBefore.length();
            }
            String cache = html.substring(index, html.length());
            int end = cache.length();
            if(StringUtils.isNotEmpty(rightBefore)) {
                end = cache.indexOf(rightBefore);
                if(end<0) {
                    throw new CustomException("返回数据找不到RB值!RB:"+rightBefore);
                }
            }
            result = cache.substring(0, end);
        }catch(Exception e){
            throw new CustomException("找不到值!LB:"+leftBefore+";RB:"+rightBefore+";异常:"+e.getMessage());
        }
        return result;
    }

}

    }

}

HttpResult.java

package com.test.http;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Serializable;

import org.apache.log4j.Logger;

import com.test.http.util.StringUtils;

public class HttpResult implements Serializable {

    private static Logger log = Logger.getLogger(HttpResult.class);

    private static final long serialVersionUID = 1L;

    /**
     * 请求过程中的链接
     */
    private String requestUrl;

    /**
     * 请求头信息
     */
    private String requestHeader;

    /**
     * 响应头信息
     */
    private String responseHeader;

    /**
     * 响应正文
     */
    private String responseBody;

    /**
     * 响应状态码
     */
    private int statusCode;

    /**
     * 响应内容编码
     */
    private String charSet;

    public String getRequestUrl() {
        return requestUrl;
    }

    public void setRequestUrl(String requestUrl) {
        this.requestUrl = requestUrl;
    }

    public String getRequestHeader() {
        return requestHeader;
    }

    public void setRequestHeader(String requestHeader) {
        this.requestHeader = requestHeader;
    }

    public String getResponseHeader() {
        return responseHeader;
    }

    public void setResponseHeader(String responseHeader) {
        this.responseHeader = responseHeader;
    }

    public String getResponseBody() {
        return responseBody;
    }

    public void setResponseBody(String responseBody) {
        this.responseBody = responseBody;
    }

    public int getStatusCode() {
        return statusCode;
    }

    public void setStatusCode(int statusCode) {
        this.statusCode = statusCode;
    }

    public String getCharSet() {
        return charSet;
    }

    public void setCharSet(String charSet) {
        this.charSet = charSet;
    }

    public String toString() {
        StringBuilder str = new StringBuilder();
        str.append("请求连接：\r\n").append(this.requestUrl);
        str.append("\r\n请求头：\r\n").append(this.requestHeader);
        str.append("\r\n响应头：\r\n").append(this.responseHeader);
        str.append("\r\n状态码：").append(this.statusCode);
        str.append("\r\n响应内容： \r\n").append(this.responseBody);
        return str.toString();
    }

    /**
     * 内容输出到日志
     * @param debugLogPath 日志文件夹路径
     * @param charSet html文件编码
     * @throws IOException
     */
    public void log(String debugLogPath) {
        String coding = "UTF-8";
        if(StringUtils.isNotEmpty(this.charSet)) {
            coding = this.charSet;
        }
        log.info("请求连接：\r\n"+this.requestUrl);
        log.info("请求头：\r\n"+this.requestHeader);
        log.info("响应头：\r\n"+this.responseHeader);
        log.info("状态码："+this.statusCode);
        if(StringUtils.isNotEmpty(debugLogPath) && responseBody.indexOf("<html")!=-1) {
            long timeMillis = System.currentTimeMillis();
            File dir = new File(debugLogPath);
            if(!dir.exists()) {
                dir.mkdirs();
            }
            String filePath = debugLogPath + File.separator + timeMillis + ".html";
            log.info("响应内容路径：" + filePath);
            File file = new File(filePath);
            try(OutputStream outputStream = new FileOutputStream(file, false);
                    OutputStreamWriter outputStreamWriter = new OutputStreamWriter(outputStream, coding)){
                outputStreamWriter.append(this.responseBody);
                outputStreamWriter.flush();
            }catch (Exception e) {
                log.info("响应内容：\r\n" + this.responseBody);
                log.info("日志目录:" + debugLogPath);
                log.error("响应内容输出到文件失败", e);
            }
        }else {
            log.info("响应内容：\r\n" + this.responseBody);
            log.info("日志目录:" + debugLogPath);
        }
    }

}

Request.java

package com.test.http;

import java.io.Serializable;
import java.util.Map;

import org.apache.http.client.HttpClient;

/**
 * Http请求实体类
 * @author youxiang
 *
 */
public class Request implements Serializable {

    private static final long serialVersionUID = 1L;

    /**
     * HttpClient对象
     */
    private HttpClient httpClient;
    /**
     * 请求链接
     */
    private String url;
    /**
     * 请求头
     */
    private Map<String, String> headers;
    /**
     * 请求参数
     */
    private Map<String, String> values;
    /**
     * 请求body内容
     */
    private String bodyData;

    public HttpClient getHttpClient() {
        return httpClient;
    }

    public void setHttpClient(HttpClient httpClient) {
        this.httpClient = httpClient;
    }

    public String getUrl() {
        return url;
    }

    public void setUrl(String url) {
        this.url = url;
    }

    public Map<String, String> getHeaders() {
        return headers;
    }

    public void setHeaders(Map<String, String> headers) {
        this.headers = headers;
    }

    public Map<String, String> getValues() {
        return values;
    }

    public void setValues(Map<String, String> values) {
        this.values = values;
    }

    public String getBodyData() {
        return bodyData;
    }

    public void setBodyData(String bodyData) {
        this.bodyData = bodyData;
    }

    public static long getSerialversionuid() {
        return serialVersionUID;
    }

}

PostRequest.java

package com.test.http;

import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;

import org.apache.http.NameValuePair;
import org.apache.http.client.HttpClient;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.entity.StringEntity;
import org.apache.http.message.BasicNameValuePair;
import org.apache.log4j.Logger;

import com.test.http.util.HttpUtils;
import com.test.http.util.StringUtils;

public class PostRequest {

    private static Logger log = Logger.getLogger(PostRequest.class);

    private PostRequest() {
        throw new IllegalStateException("Utility class");
    }

    /**
     * POST请求
     * @param request
     *      {
     *          httpClient : HttpClient对象
     *          url : 请求链接
     *          headers : 请求头
     *          values : 请求参数
     *          bodyData : body请求参数
     *      }
     * @return HttpClientContext
     * @throws IOException
     */
    public static HttpClientContext execute(Request request) throws IOException {
        HttpClientContext httpContext = HttpClientContext.create();
        HttpClient httpClient = request.getHttpClient();
        if(httpClient==null) {
            log.info("HttpClient对象为空，自动获取新HttpClient对象");
            httpClient = HttpUtils.getHttpClient();
        }
        HttpPost post = new HttpPost(request.getUrl());
        //设置请求头
        PostRequest.setHeader(post, request.getHeaders());
        PostRequest.setValue(post, request.getValues());
        PostRequest.setBodyValue(post, request.getBodyData());
        httpClient.execute(post, httpContext);
        return httpContext;
    }

    /**
     * post设置请求头
     * @param post
     * @param headers
     * @throws Exception
     */
    private static void setHeader(HttpPost post, Map<String, String> headers) {
        if (headers != null && !headers.isEmpty()) {
            if(!headers.containsKey("User-Agent")){
                post.addHeader("User-Agent", HttpUtils.USER_AGENT_VALUE);
            }
            for(Map.Entry<String, String> header_entry : headers.entrySet()){
                post.addHeader(header_entry.getKey(), header_entry.getValue());
            }
        }
    }


    /**
     * post设置数据
     * @param post
     * @param values
     * @throws UnsupportedEncodingException 
     * @throws Exception
     */
    private static void setValue(HttpPost post, Map<String, String> values) throws UnsupportedEncodingException {
        if (values != null && !values.isEmpty()) {
            List<NameValuePair> nvps = new ArrayList<>();
            for(Map.Entry<String, String> value_entry : values.entrySet()){
                nvps.add(new BasicNameValuePair(value_entry.getKey(), value_entry.getValue()));
            }
            post.setEntity(new UrlEncodedFormEntity(nvps, HttpUtils.HTTP_PARAMS_CODING));
        }
    }

    /**
     * postBody设置数据
     * @param post
     * @param bodyValue
     * @throws Exception
     */
    private static void setBodyValue(HttpPost post, String bodyValue) {
        if(StringUtils.isNotEmpty(bodyValue)) {
            post.addHeader("Accept", "application/json");
            post.addHeader("Content-Type", "application/json; charset="+HttpUtils.HTTP_PARAMS_CODING);
            StringEntity entity = new StringEntity(bodyValue, Charset.forName(HttpUtils.HTTP_PARAMS_CODING));
            entity.setContentEncoding(HttpUtils.HTTP_PARAMS_CODING);
            entity.setContentType("application/json");// 设置为 json数据
            post.setEntity(entity);
        }
    }

}

GetRequest.java

package com.test.http;

import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.Map;

import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.log4j.Logger;

import com.test.http.util.HttpUtils;

public class GetRequest {

    private static Logger log = Logger.getLogger(GetRequest.class);

    private GetRequest() {
        throw new IllegalStateException("Utility class");
    }

    /**
     * GET请求
     * @param request
     *      {
     *          httpClient : HttpClient对象
     *          url : 请求链接
     *          headers : 请求头
     *          values : 请求参数
     *      }
     * @return HttpClientContext
     * @throws IOException
     */
    public static HttpClientContext execute(Request request) throws IOException {
        HttpClientContext httpContext = HttpClientContext.create();
        HttpClient httpClient = request.getHttpClient();
        if(httpClient==null) {
            log.info("HttpClient对象为空，自动获取新HttpClient对象");
            httpClient = HttpUtils.getHttpClient();
        }
        Map<String, String> values = request.getValues();
        String url=GetRequest.setValue(request.getUrl(), values);
        HttpGet get = new HttpGet(url);
        //设置请求头
        GetRequest.setHeader(get, request.getHeaders());
        httpClient.execute(get, httpContext);
        return httpContext;
    }

    /**
     * get设置请求头 
     * @param get
     * @param headers
     * @throws Exception
     */
    private static void setHeader(HttpGet get, Map<String, String> headers) {
        if (headers != null && !headers.isEmpty()) {
            if(!headers.containsKey("User-Agent")){
                get.addHeader("User-Agent", HttpUtils.USER_AGENT_VALUE);
            }
            for(Map.Entry<String, String> header_entry : headers.entrySet()){
                get.addHeader(header_entry.getKey(), header_entry.getValue());
            }
        }
    }

    /**
     * get设置数据
     * @param get
     * @param values
     * @throws UnsupportedEncodingException 
     * @throws Exception
     */
    private static String setValue(String url, Map<String, String> values) throws UnsupportedEncodingException {
        StringBuilder param = new StringBuilder(url);
        if (values != null && !values.isEmpty()) {
            for(Map.Entry<String, String> value_entry : values.entrySet()){
                if(param.length()==0 || param.indexOf("?") == -1){
                    param.append("?"+URLEncoder.encode(value_entry.getKey(), HttpUtils.HTTP_PARAMS_CODING)+"="+URLEncoder.encode(value_entry.getValue(), HttpUtils.HTTP_PARAMS_CODING));
                }else{
                    param.append("&"+URLEncoder.encode(value_entry.getKey(), HttpUtils.HTTP_PARAMS_CODING)+"="+URLEncoder.encode(value_entry.getValue(), HttpUtils.HTTP_PARAMS_CODING));
                }
            }
        }
        return param.toString();
    }

}

测试方法

public static void main(String[] args) throws IOException {
    HttpClient httpclient = HttpUtils.getHttpClient();
    //String url = "http://192.168.30.122:8080/risk-war";
    String url = "https://www.baidu.com";
    Map<String, String> headers=new HashMap<String, String>();
    //headers.put("Cookie", "JSESSIONID=50CA8625B613A4A335D971B8A3500D17");
    Map<String, String> values=new HashMap<String, String>();
    values.put("username", "youxiang");
    values.put("password", "123456");
    values.put("referer", "");
    Request request = new Request();
    request.setHeaders(headers);
    request.setHttpClient(httpclient);
    request.setUrl(url);
    request.setValues(values);
    HttpClientContext httpContext = GetRequest.execute(request);
    HttpResult result = HttpUtils.getHttpInfo(httpContext);
    System.out.println(result.toString());
}