httpclient的使用

准备步骤

1. 打开谷歌浏览器，按F12键，访问链接
2. 查看Network，找到访问后台的链接请求，点开
   
3. 我们一般看请求方式、请求头里面的Content-Type和Cookie以及请求参数部分，根据这些内容去使用httpclient爬取我们需要的数据，当然如果你有postman工具可以去尝试更好

httpclient的使用

环境搭建

	<!-- https://mvnrepository.com/artifact/org.apache.httpcomponents/httpclient -->
	<dependency>
	    <groupId>org.apache.httpcomponents</groupId>
	    <artifactId>httpclient</artifactId>
	    <version>4.5.6</version>
	</dependency>
	<!-- https://mvnrepository.com/artifact/org.apache.httpcomponents/httpcore -->
	<dependency>
	    <groupId>org.apache.httpcomponents</groupId>
	    <artifactId>httpcore</artifactId>
	    <version>4.4.11</version>
	</dependency>
	<!-- https://mvnrepository.com/artifact/commons-io/commons-io -->
	<dependency>
	    <groupId>commons-io</groupId>
	    <artifactId>commons-io</artifactId>
	    <version>2.4</version>
	</dependency>
	<!-- https://mvnrepository.com/artifact/commons-logging/commons-logging -->
	<dependency>
	    <groupId>commons-logging</groupId>
	    <artifactId>commons-logging</artifactId>
	    <version>1.1.1</version>
	</dependency>

HttpClientUtil

import org.apache.commons.io.Charsets;
import org.apache.http.HttpEntityEnclosingRequest;
import org.apache.http.HttpRequest;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.HttpRequestRetryHandler;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.config.SocketConfig;
import org.apache.http.conn.ConnectTimeoutException;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;

/**
 * 使用HttpClient发送和接收Http请求
 *
 * @author manzhizhen
 *
 */
public class HttpUtil {

    private static HttpClient httpClient;
    // 最大连接数
    private static final int MAX_CONNECTION = 100;
    // 每个route能使用的最大连接数，一般和MAX_CONNECTION取值一样
    private static final int MAX_CONCURRENT_CONNECTIONS = 100;
    // 建立连接的超时时间，单位毫秒
    private static final int CONNECTION_TIME_OUT = 100000;
    // 请求超时时间，单位毫秒
    private static final int REQUEST_TIME_OUT = 100000;
    // 最大失败重试次数
    private static final int MAX_FAIL_RETRY_COUNT = 3;
    // 请求配置，可以复用
    private static RequestConfig requestConfig;

    static {
        SocketConfig socketConfig = SocketConfig.custom()
                .setSoTimeout(REQUEST_TIME_OUT).setSoKeepAlive(true)
                .setTcpNoDelay(true).build();

        requestConfig = RequestConfig.custom()
                .setSocketTimeout(REQUEST_TIME_OUT)
                .setConnectTimeout(CONNECTION_TIME_OUT).build();
        /**
         * 每个默认的 ClientConnectionPoolManager 实现将给每个route创建不超过2个并发连接，最多20个连接总数。
         */
        PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager();
        connManager.setMaxTotal(MAX_CONNECTION);
        connManager.setDefaultMaxPerRoute(MAX_CONCURRENT_CONNECTIONS);
        connManager.setDefaultSocketConfig(socketConfig);

        httpClient = HttpClients.custom().setConnectionManager(connManager)
                // 添加重试处理器
                .setRetryHandler(new MyHttpRequestRetryHandler()).build();
    }


    /**
     * post请求
     *
     * @param url
     * @param paramMap
     * @param headers
     * @return
     * @throws Exception
     */
    public static String post(String url, Map<String, String> paramMap,
                              Map<String, String> headers) throws Exception {
        URIBuilder uriBuilder = new URIBuilder(url);
        if (paramMap != null) {
            // 添加请求参数
            for (Entry<String, String> entry : paramMap.entrySet()) {
                uriBuilder.addParameter(entry.getKey(), entry.getValue());
            }
        }

        HttpPost httpPost = new HttpPost(uriBuilder.build());
        if (headers != null) {
            // 添加请求首部
            for (String key : headers.keySet()) {
                httpPost.addHeader(key,headers.get(key));
            }
        }

        httpPost.setConfig(requestConfig);

        // 执行请求
        HttpResponse response = httpClient.execute(httpPost);

        return EntityUtils.toString(response.getEntity(), Charsets.UTF_8);
    }

    /**
     * post请求，不带请求首部
     *
     * @param url
     * @param paramMap
     * @return
     * @throws Exception
     */
    public static String post(String url, Map<String, String> paramMap)
            throws Exception {

        return post(url, paramMap, null);
    }

    /**
     * get请求
     *
     * @param url
     * @param paramMap
     * @param headers
     * @return
     * @throws Exception
     */
    public static String get(String url, Map<String, String> paramMap,
                             Map<String, String> headers) throws Exception {
        URIBuilder uriBuilder = new URIBuilder(url);
        if (paramMap != null) {
            // 添加请求参数
            for (Entry<String, String> entry : paramMap.entrySet()) {
                uriBuilder.addParameter(entry.getKey(), entry.getValue());
            }
        }

        HttpGet httpGet = new HttpGet(uriBuilder.build());
        if (headers != null) {
            // 添加请求首部
            for (String key : headers.keySet()) {
                httpGet.addHeader(key,headers.get(key));
            }
        }

        httpGet.setConfig(requestConfig);

        // 执行请求
        HttpResponse response = httpClient.execute(httpGet);

        return EntityUtils.toString(response.getEntity(), Charsets.UTF_8);
    }

    /**
     * get请求，不带请求首部
     *
     * @param url
     * @param paramMap
     * @return
     * @throws Exception
     */
    public static String get(String url, Map<String, String> paramMap) throws Exception {

        return get(url, paramMap, null);
    }

    /**
     * 请求重试处理器
     * @author manzhizhen
     *
     */
    private static class MyHttpRequestRetryHandler implements HttpRequestRetryHandler {

        @Override
        public boolean retryRequest(IOException exception, int executionCount,
                                    HttpContext context) {
            if (executionCount >= MAX_FAIL_RETRY_COUNT) {
                return false;
            }

            if (exception instanceof InterruptedIOException) {
                // 超时
                return false;
            }
            if (exception instanceof UnknownHostException) {
                // 未知主机
                return false;
            }
            if (exception instanceof ConnectTimeoutException) {
                // 连接被拒绝
                return false;
            }
            if (exception instanceof SSLException) {
                // SSL handshake exception
                return false;
            }

            HttpClientContext clientContext = HttpClientContext.adapt(context);
            HttpRequest request = clientContext.getRequest();
            boolean idempotent = !(request instanceof HttpEntityEnclosingRequest);
            if (idempotent) {
                // 如果请求被认为是幂等的，则重试
                return true;
            }

            return false;
        }
    }

    //get请求保存图片
    public static void getSaveImg(String url,HttpServletResponse response) throws Exception {

        URIBuilder uriBuilder = new URIBuilder(url);
        HttpGet httpGet = new HttpGet(uriBuilder.build());
        httpGet.setHeader("Content-Type","application/json");
        httpGet.setConfig(requestConfig);
		
        // 执行请求
        HttpResponse resp = httpClient.execute(httpGet);
        if (resp.getStatusLine().getStatusCode() == 200) {
            byte[] data = EntityUtils.toByteArray(resp.getEntity());
            
            //保存图片到本地
            File file = new File("c:/1.png");
            //如果文件不存在，就创建一个
			if (!file.exists()) {
    			file.createNewFile();
   			}
   			FileOutputStream os = new FileOutputStream(file);
   			os.write(data);
   			os.flush();
   			os.close();

			//将图片显示到另外一个页面
			OutputStream os = response.getOutputStream();
        	os.write(data);
        	os.close();
        }
    }
}