HttpClient
项目结构
配置文件
-
pom.xml
<?xml version="1.0" encoding="UTF-8"?> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> <groupId>cn.xiaoge</groupId> <artifactId>xiaoge-crawler-first</artifactId> <version>1.0-SNAPSHOT</version> <dependencies> <!-- 自动的抓取数据的jar包, httpclient --> <dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpclient</artifactId> <version>4.5.2</version> </dependency> <!-- 日志信息 --> <dependency> <groupId>org.slf4j</groupId> <artifactId>slf4j-log4j12</artifactId> <version>1.7.25</version> <!--<scope>test</scope>--> </dependency> </dependencies> </project>
-
log4j.properties
log4j.rootLogger=DEBUG,A1 log4j.logger.cn.itcast=DEBUG # 在控制台显示日志 log4j.appender.A1=org.apache.log4j.ConsoleAppender log4j.appender.A1.layout=org.apache.log4j.PatternLayout log4j.appender.A1.layout.ConversionPattern=%-d{yyyy-MM-dd HH:mm:ss,SSS} [%t] [%c]-[%p] %m%n
操作类
-
HttpGetTest(不带参Get)
package com.xiaoge.crawler.test; import org.apache.http.HttpEntity; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.util.EntityUtils; import java.io.IOException; /** * @Author: 潇哥 * @DateTime: 2020/9/21 下午4:35 * @Description: TODO */ public class HttpGetTest { public static void main(String[] args) { // 1. 创建HttpClient对象 CloseableHttpClient httpClient = HttpClients.createDefault(); // 2. 创建HttpGet对象, 设置url访问地址 HttpGet httpGet = new HttpGet("https://www.csdn.net/"); CloseableHttpResponse httpResponse = null; try{ // 3. 使用HttpClient发起请求, 获取response httpResponse = httpClient.execute(httpGet); // 4. 解析响应 if (httpResponse.getStatusLine().getStatusCode() == 200) { // 获取响应体 HttpEntity httpEntity = httpResponse.getEntity(); String content = EntityUtils.toString(httpEntity, "utf8"); System.out.println(content.length()); } } catch (IOException e) { e.printStackTrace(); } finally { // 关闭httpResponse if (httpResponse != null) { try { httpResponse.close(); } catch (IOException e) { e.printStackTrace(); } } // 关闭httpClient if (httpClient != null) { try { httpClient.close(); } catch (IOException e) { e.printStackTrace(); } } } } }
-
HttpGetParamTest(带参Get)
package com.xiaoge.crawler.test; import org.apache.http.HttpEntity; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.utils.URIBuilder; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.util.EntityUtils; import java.io.IOException; /** * @Author: 潇哥 * @DateTime: 2020/9/21 下午4:35 * @Description: TODO */ public class HttpGetParamTest { public static void main(String[] args) throws Exception { // 1. 创建HttpClient对象 CloseableHttpClient httpClient = HttpClients.createDefault(); // 设置请求地址是: https://so.csdn.net/so/search/s.do?q=java // 创建URIBuilder URIBuilder uriBuilder = new URIBuilder("https://so.csdn.net/so/search/s.do"); // 设置参数, 如果是多个参数继续setParameter uriBuilder.setParameter("q", "java"); // 2. 创建HttpGet对象, 设置url访问地址 HttpGet httpGet = new HttpGet(uriBuilder.build()); CloseableHttpResponse httpResponse = null; try{ // 3. 使用HttpClient发起请求, 获取response httpResponse = httpClient.execute(httpGet); // 4. 解析响应 if (httpResponse.getStatusLine().getStatusCode() == 200) { // 获取响应体 HttpEntity httpEntity = httpResponse.getEntity(); String content = EntityUtils.toString(httpEntity, "utf8"); System.out.println(content.length()); } } catch (IOException e) { e.printStackTrace(); } finally { // 关闭httpResponse if (httpResponse != null) { try { httpResponse.close(); } catch (IOException e) { e.printStackTrace(); } } // 关闭httpClient if (httpClient != null) { try { httpClient.close(); } catch (IOException e) { e.printStackTrace(); } } } } }
-
HttpPostTest(不带参post)
package com.xiaoge.crawler.test; import org.apache.http.HttpEntity; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpPost; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.util.EntityUtils; import java.io.IOException; /** * @Author: 潇哥 * @DateTime: 2020/9/21 下午4:35 * @Description: TODO */ public class HttpPostTest { public static void main(String[] args) { // 1. 创建HttpClient对象 CloseableHttpClient httpClient = HttpClients.createDefault(); // 2. 创建HttpPost对象, 设置url访问地址 HttpPost httpPost = new HttpPost("https://www.csdn.net/"); CloseableHttpResponse httpResponse = null; try{ // 3. 使用HttpClient发起请求, 获取response httpResponse = httpClient.execute(httpPost); // 4. 解析响应 if (httpResponse.getStatusLine().getStatusCode() == 200) { // 获取响应体 HttpEntity httpEntity = httpResponse.getEntity(); String content = EntityUtils.toString(httpEntity, "utf8"); System.out.println(content.length()); } } catch (IOException e) { e.printStackTrace(); } finally { // 关闭httpResponse if (httpResponse != null) { try { httpResponse.close(); } catch (IOException e) { e.printStackTrace(); } } // 关闭httpClient if (httpClient != null) { try { httpClient.close(); } catch (IOException e) { e.printStackTrace(); } } } } }
-
HttpPostParamTest(带参post)
package com.xiaoge.crawler.test; import org.apache.http.HttpEntity; import org.apache.http.NameValuePair; import org.apache.http.client.entity.UrlEncodedFormEntity; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpPost; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.message.BasicNameValuePair; import org.apache.http.util.EntityUtils; import java.io.IOException; import java.util.ArrayList; import java.util.List; /** * @Author: 潇哥 * @DateTime: 2020/9/21 下午4:35 * @Description: TODO */ public class HttpPostParamTest { public static void main(String[] args) throws Exception { // 1. 创建HttpClient对象 CloseableHttpClient httpClient = HttpClients.createDefault(); // 2. 创建HttpPost对象, 设置url访问地址 HttpPost httpPost = new HttpPost("https://so.csdn.net/so/search/s.do"); // 声明List集合, 封装表单中的参数 List<NameValuePair> params = new ArrayList<NameValuePair>(); // 设置请求地址是: https://so.csdn.net/so/search/s.do?q=java params.add(new BasicNameValuePair("q", "java")); // 创建表单的Entity对象, 第一个参数就是封装好的表单数据, 第二个参数就是编码 UrlEncodedFormEntity formEntity = new UrlEncodedFormEntity(params, "utf8"); // 设置表单的Entity对象到Post请求中 httpPost.setEntity(formEntity); CloseableHttpResponse httpResponse = null; try{ // 3. 使用HttpClient发起请求, 获取response httpResponse = httpClient.execute(httpPost); // 4. 解析响应 if (httpResponse.getStatusLine().getStatusCode() == 200) { // 获取响应体 HttpEntity httpEntity = httpResponse.getEntity(); String content = EntityUtils.toString(httpEntity, "utf8"); System.out.println(content.length()); } } catch (IOException e) { e.printStackTrace(); } finally { // 关闭httpResponse if (httpResponse != null) { try { httpResponse.close(); } catch (IOException e) { e.printStackTrace(); } } // 关闭httpClient if (httpClient != null) { try { httpClient.close(); } catch (IOException e) { e.printStackTrace(); } } } } }
-
连接池管理器(管理httpClient)
package com.xiaoge.crawler.test; import org.apache.http.HttpEntity; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.util.EntityUtils; import java.io.IOException; /** * @Author: 潇哥 * @DateTime: 2020/9/21 下午5:27 * @Description: TODO */ public class HttpClientPoolTest { public static void main(String[] args) { // 创建连接池管理器 PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(); // 设置最大连接数 cm.setMaxTotal(100); // 设置每个主机的最大连接数, (意思是每个网站只能链接10个, 因为每一个网站的域名就是一个主机) cm.setDefaultMaxPerRoute(10); // 使用连接池管理器发送请求 doGet(cm); doGet(cm); } private static void doGet(PoolingHttpClientConnectionManager cm) { // 不是每次创建新的HttpClient, 而是从连接池中获取HttpClient对象 CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(cm).build(); HttpGet httpGet = new HttpGet("https://www.csdn.net/"); CloseableHttpResponse httpResponse = null; try { httpResponse = httpClient.execute(httpGet); if (httpResponse.getStatusLine().getStatusCode() == 200) { HttpEntity httpEntity = httpResponse.getEntity(); String content = EntityUtils.toString(httpEntity, "utf8"); System.out.println(content.length()); } } catch (IOException e) { e.printStackTrace(); } finally { // 关闭httpResponse if (httpResponse != null) { try { httpResponse.close(); } catch (IOException e) { e.printStackTrace(); } } // 不能关闭HttpClient, 由连接池管理HttpClient // httpClient.close(); } } }
-
HttpConfigTest(设置请求信息)
package com.xiaoge.crawler.test; import org.apache.http.HttpEntity; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.util.EntityUtils; import java.io.IOException; /** * @Author: 潇哥 * @DateTime: 2020/9/21 下午4:35 * @Description: TODO */ public class HttpConfigTest { public static void main(String[] args) { // 1. 创建HttpClient对象 CloseableHttpClient httpClient = HttpClients.createDefault(); // 2. 创建HttpGet对象, 设置url访问地址 HttpGet httpGet = new HttpGet("https://www.csdn.net/"); // 配置请求信息 RequestConfig config = RequestConfig.custom().setConnectTimeout(1000)// 创建连接的最长时间, 单位是毫秒 .setConnectionRequestTimeout(500) // 设置获取链接的最长时间, 单位毫秒 .setSocketTimeout(10 * 1000) // 设置数据传输的最长时间, 单位毫秒 .build(); // 给请求设置请求信息 httpGet.setConfig(config); CloseableHttpResponse httpResponse = null; try{ // 3. 使用HttpClient发起请求, 获取response httpResponse = httpClient.execute(httpGet); // 4. 解析响应 if (httpResponse.getStatusLine().getStatusCode() == 200) { // 获取响应体 HttpEntity httpEntity = httpResponse.getEntity(); String content = EntityUtils.toString(httpEntity, "utf8"); System.out.println(content.length()); } } catch (IOException e) { e.printStackTrace(); } finally { // 关闭httpResponse if (httpResponse != null) { try { httpResponse.close(); } catch (IOException e) { e.printStackTrace(); } } // 关闭httpClient if (httpClient != null) { try { httpClient.close(); } catch (IOException e) { e.printStackTrace(); } } } } }