HttpClient 提供高效的,最新的,功能丰富的支持HTTP协议的客户端编程工具,并且支持HTTP协议最新版本和建议
HttpClient可以模拟浏览器请求第三方站点url,然后响应获取网页数据。
引入依赖
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.httpclient</groupId>
<artifactId>HttpClientDemo</artifactId>
<version>0.0.1-SNAPSHOT</version>
<dependencies>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.2</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.5</version>
</dependency>
</dependencies>
</project>
获得httpClient客户端实例,执行get和post请求
post可以提交form
get直接url
execute 发送请求
import java.io.IOException;
import org.apache.http.HttpEntity;
import org.apache.http.ParseException;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
public class HelloWorld {
public static void main(String[] args) {
CloseableHttpClient httpClient=HttpClients.createDefault(); // 创建httpClient实例
HttpGet httpGet=new HttpGet("http://www.tuicool.com/"); // 创建httpget实例
CloseableHttpResponse response=null;
try {
response=httpClient.execute(httpGet); // 执行http get请求
} catch (ClientProtocolException e) { // http协议异常
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) { // io异常
// TODO Auto-generated catch block
e.printStackTrace();
}
HttpEntity entity=response.getEntity(); // 获取返回实体
try {
System.out.println("网页内容:"+EntityUtils.toString(entity, "utf-8")); // 获取网页内容
} catch (ParseException e) { // 解析异常
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) { // io异常
// TODO Auto-generated catch block
e.printStackTrace();
}
try {
response.close(); // response关闭
} catch (IOException e) { // io异常
// TODO Auto-generated catch block
e.printStackTrace();
}
try {
httpClient.close(); // httpClient关闭
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
抓取网页
设置请求头消息 User-Agent 模拟浏览器
获取响应内容类型 Content-Type 响应状态 Status(200正常,403拒绝,500服务器报错,400未找到页面)
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
public class Demo {
public static void main(String[] args)throws Exception {
CloseableHttpClient httpClient=HttpClients.createDefault(); // 创建httpClient实例
HttpGet httpGet=new HttpGet("http://www.tuicool.com/"); // 创建httpget实例
httpGet.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:50.0) Gecko/20100101 Firefox/50.0");
CloseableHttpResponse response=httpClient.execute(httpGet); // 执行http get请求
System.out.println("Status:"+response.getStatusLine().getStatusCode());
HttpEntity entity=response.getEntity(); // 获取返回实体
System.out.println("网页内容:"+EntityUtils.toString(entity, "utf-8")); // 获取网页内容
System.out.println("Content-Type:"+entity.getContentType().getValue());
response.close(); // response关闭
httpClient.close(); // httpClient关闭
}
}
HttpClient 通过流抓取图片
import java.io.File;
import java.io.InputStream;
import org.apache.commons.io.FileUtils;
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
public class Demo1 {
public static void main(String[] args)throws Exception {
CloseableHttpClient httpClient=HttpClients.createDefault(); // 创建httpClient实例
HttpGet httpGet=new HttpGet("http://www.java1234.com/gg/dljd4.gif"); // 创建httpget实例
httpGet.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:50.0) Gecko/20100101 Firefox/50.0");
CloseableHttpResponse response=httpClient.execute(httpGet); // 执行http get请求
HttpEntity entity=response.getEntity(); // 获取返回实体
if(entity!=null){
System.out.println("ContentType:"+entity.getContentType().getValue());
InputStream inputStream=entity.getContent();
FileUtils.copyToFile(inputStream, new File("C://dljd4.gif"));
}
response.close(); // response关闭
httpClient.close(); // httpClient关闭
}
}
HttpClient 使用代理ip
代理ip种类,透明代理,匿名代理,混淆代理,高匿代理
一般用高匿代理
HttpHost 代理主机
custom().setProxy(proxy).build() 设置代理ip返回构造
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
public class Demo01 {
public static void main(String[] args)throws Exception {
CloseableHttpClient httpClient=HttpClients.createDefault(); // 创建httpClient实例
HttpGet httpGet=new HttpGet("http://www.tuicool.com/"); // 创建httpget实例
HttpHost proxy=new HttpHost("175.155.213.235", 9999);
RequestConfig config=RequestConfig.custom().setProxy(proxy).build();
httpGet.setConfig(config);
httpGet.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:50.0) Gecko/20100101 Firefox/50.0");
CloseableHttpResponse response=httpClient.execute(httpGet); // 执行http get请求
HttpEntity entity=response.getEntity(); // 获取返回实体
System.out.println("网页内容:"+EntityUtils.toString(entity, "utf-8")); // 获取网页内容
response.close(); // response关闭
httpClient.close(); // httpClient关闭
}
}
Http连接超时和读取超时
import org.apache.http.HttpEntity;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
public class Demo {
public static void main(String[] args)throws Exception {
CloseableHttpClient httpClient=HttpClients.createDefault(); // 创建httpClient实例
HttpGet httpGet=new HttpGet("http://central.maven.org/maven2/"); // 创建httpget实例
RequestConfig config=RequestConfig.custom()
.setConnectTimeout(10000) // 设置连接超时时间 10秒钟
.setSocketTimeout(20000) // 设置读取超时时间10秒钟
.build();
httpGet.setConfig(config);
httpGet.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:50.0) Gecko/20100101 Firefox/50.0");
CloseableHttpResponse response=httpClient.execute(httpGet); // 执行http get请求
HttpEntity entity=response.getEntity(); // 获取返回实体
System.out.println("网页内容:"+EntityUtils.toString(entity, "utf-8")); // 获取网页内容
response.close(); // response关闭
httpClient.close(); // httpClient关闭
}
}