HttpClient（访问API，爬取数据）

一、环境

<dependencies>
    <!-- HttpClient -->
    <dependency>
        <groupId>org.apache.httpcomponents</groupId>
        <artifactId>httpclient</artifactId>
        <version>4.5.3</version>
    </dependency>
</dependencies>

二、代码

2.1HttpGet

@Test
public void httpGetTest() {
    //1.使用httpclient访问
    CloseableHttpClient httpClient = HttpClients.createDefault();
    HttpGet httpGet = new HttpGet("http://www.itcast.cn");
    CloseableHttpResponse response = null;
    try {
        response = httpClient.execute(httpGet);
        //2.解析响应
        if (response.getStatusLine().getStatusCode() == 200) {
            String content = EntityUtils.toString(response.getEntity(), "utf-8");
            System.out.println(content);
        }
    } catch (IOException e) {
        e.printStackTrace();
    } finally {
        try {
            if (response != null) {
                response.close();
            }
            if (httpClient != null) {
                httpClient.close();
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}

2.2HttpGet带参数

@Test
public void httpGetParamTest() throws URISyntaxException {
    CloseableHttpClient httpClient = HttpClients.createDefault();
    //参数
    URIBuilder uriBuilder = new URIBuilder("https://so.csdn.net/so/search/s.do");
    uriBuilder.setParameter("q", "啊哈哈");
    HttpGet httpGet = new HttpGet(uriBuilder.build());
    CloseableHttpResponse response = null;
    try {
        response = httpClient.execute(httpGet);
        
        if (response.getStatusLine().getStatusCode() == 200) {
            String content = EntityUtils.toString(response.getEntity(), "utf-8");
            System.out.println(content);
        }
    } catch (IOException e) {
        e.printStackTrace();
    } finally {
        try {
            if (response != null) {
                response.close();
            }
            if (httpClient != null) {
                httpClient.close();
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}

2.3HttpPost

@Test
public void httpPostTest() {
    
    CloseableHttpClient httpClient = HttpClients.createDefault();
    HttpPost httpPost = new HttpPost("http://www.itcast.cn");
    CloseableHttpResponse response = null;
    try {
        response = httpClient.execute(httpPost);
        
        if (response.getStatusLine().getStatusCode() == 200) {
            String content = EntityUtils.toString(response.getEntity(), "utf-8");
            System.out.println(content);
        }
    } catch (IOException e) {
        e.printStackTrace();
    } finally {
        try {
            if (response != null) {
                response.close();
            }
            if (httpClient != null) {
                httpClient.close();
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}

2.4HttpPost带参

@Test
public void httpPostParamTest() throws UnsupportedEncodingException {
    
    CloseableHttpClient httpClient = HttpClients.createDefault();
    HttpPost httpPost = new HttpPost("http://yun.itheima.com/search");

    //声明List集合，封装表单中的参数
    List<NameValuePair> params = new ArrayList<>();
    params.add(new BasicNameValuePair("keys", "java"));


    //创建表单的Entity对象
    UrlEncodedFormEntity formEntity = new UrlEncodedFormEntity(params, "UTF-8");
    //设置表单的Entity对象到Post请求中
    httpPost.setEntity(formEntity);
    CloseableHttpResponse response = null;
    try {
        response = httpClient.execute(httpPost);
        
        if (response.getStatusLine().getStatusCode() == 200) {
            String content = EntityUtils.toString(response.getEntity(), "utf-8");
            System.out.println(content);
        }
    } catch (IOException e) {
        e.printStackTrace();
    } finally {
        try {
            if (response != null) {
                response.close();
            }
            if (httpClient != null) {
                httpClient.close();
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}

2.5连接池

@Test
public void httpClientPoolTest() {

    //创建连接池管理器
    PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();

    //设置最大连接数
    cm.setMaxTotal(100);

    //设置每个主机的最大连接数
    cm.setDefaultMaxPerRoute(10);

    //使用连接池管理器发起请求
    doGet(cm);
    doGet(cm);
}

//httpClientPoolTest()使用
public void doGet(PoolingHttpClientConnectionManager cm) {
    //不是每次都创建新的httpClient,而是从连接池中获取HttpClient对象
    CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(cm).build();
    CloseableHttpResponse response = null;
    try {
        response = httpClient.execute(new HttpGet("http://www.itcast.cn"));
        if (response.getStatusLine().getStatusCode() == 200) {
            String content = EntityUtils.toString(response.getEntity(), "UTF-8");
            System.out.println(content.length());
        }
    } catch (IOException e) {
        e.printStackTrace();
    } finally {
        if (response != null) {
            try {
                response.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }
}

2.6参数

@Test
public void httpConfigTest() {
    CloseableHttpClient httpClient = HttpClients.createDefault();
    HttpGet httpGet = new HttpGet("http://www.itcast.cn/");

    //设置请求参数
    RequestConfig requestConfig = RequestConfig.custom()
        .setConnectTimeout(1000)//设置创建连接的最长时间,单位毫秒
        .setConnectionRequestTimeout(500)//设置获取连接的最长时间
        .setSocketTimeout(10 * 1000)//设置数据传输的最长时间
        .build();

    httpGet.setConfig(requestConfig);
    CloseableHttpResponse response = null;
    try {
        response = httpClient.execute(httpGet);
        if (response.getStatusLine().getStatusCode() == 200) {
            String content = EntityUtils.toString(response.getEntity(), "UTF-8");
            System.out.println(content);
        }
    } catch (Exception e) {
        e.printStackTrace();
    } finally {
        try {
            if (response == null) {
                response.close();
            }
            httpClient.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}

2.7京东反扒解决

设置请求头

httpGet.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:62.0) Gecko/20100101 Firefox/62.0");