编写不易,转载请注明(http://shihlei.iteye.com/blog/2067688)!
一 概述
HttpClient是最常见的Http请求工具,很好的封装 Http请求,响应,Cookies操作,适合网络爬虫处理静态页面及其他资源。
HttpCient能自动处理302客户端重定向,这个很喜欢!
HttpClient4.3.3 API较之前的版本有不少改变,本文描述请求响应,设置cookies,设置代理等方法。
二 版本
<dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpclient</artifactId> <version>4.3.3</version> </dependency>
三 样例
(1)请求响应
/**
* 发送get请求
*
* @param url
* @throws Exception
*/
public byte[] sendGetRequest(String url) throws Exception {
HttpClient httpClient = HttpClients.createDefault();
HttpGet httpGet = new HttpGet(url);
System.out.println("Send Get Request : " + url);
return sendRequest(httpClient,httpGet);
}
/**
* 发送post 请求
*
* @param request
* @throws Exception
*/
public byte[] sendPostRequest(String url, Map<String, String> params) throws Exception {
HttpClient httpClient = HttpClients.createDefault();
HttpPost httpPost = new HttpPost(url);
// 设置参数
if (params != null && params.size() > 0) {
List<NameValuePair> nvps = new ArrayList<NameValuePair>();
for (Entry<String, String> param : params.entrySet()) {
nvps.add(new BasicNameValuePair(param.getKey(), param.getValue()));
}
httpPost.setEntity(new UrlEncodedFormEntity(nvps));
}
System.out.println("Send Post Request : " + url + " , " + params);
return sendRequest(httpClient,httpPost);
}
// 底层请求获得响应数据
private byte[] sendRequest(HttpClient httpClient ,HttpRequestBase request) throws Exception {
CloseableHttpResponse response = httpclient.execute(request);
// 获取返回的状态列表
StatusLine statusLine = response.getStatusLine();
System.out.println("StatusLine : " + statusLine);
if (statusLine.getStatusCode() == HttpStatus.SC_OK) {
try {
// 获取response entity
HttpEntity entity = response.getEntity();
System.out.println("getContentType : " + entity.getContentType().getValue());
System.out.println("getContentEncoding : " + entity.getContentEncoding().getValue());
// 读取响应内容
byte[] responseBody = EntityUtils.toByteArray(entity);
// 关闭响应流
EntityUtils.consume(entity);
return responseBody;
} finally {
response.close();
}
}
return null;
}
(2)超时和重定向
在execute前针对Request设置超时的Config
//设置超时
private void setTimeOut(HttpRequestBase request, int timeout) {
RequestConfig requestConfig = RequestConfig.custom().setConnectionRequestTimeout(timeout)
.setConnectTimeout(timeout).setSocketTimeout(timeout).setRedirectsEnabled(true).build();
request.setConfig(requestConfig);
}
(3)代理
在创建HttpClient实例的时候,设置代理
//获得设置代理的httpClent实例
private CloseableHttpClient getHttpClientWithProxy(HttpProxy proxy){
HttpHost proxyHost = new HttpHost(proxy.getHost(), proxy.getPort());
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(new AuthScope(proxy.getHost(), proxy.getPort()),
new UsernamePasswordCredentials(proxy.getUser(), proxy.getPassword()));
return HttpClients.custom().setProxy(proxyHost).setDefaultCredentialsProvider(credentialsProvider)
.build();
}
这里有个辅助类:
package x.http.simple;
/**
* Http代理
*
* @author shilei
*
*/
public class HttpProxy {
private String proxy = "http";
private String host;
private int port;
private String user;
private String password;
public String getProxy() {
return proxy;
}
public void setProxy(String proxy) {
this.proxy = proxy;
}
public String getHost() {
return host;
}
public void setHost(String host) {
this.host = host;
}
public int getPort() {
return port;
}
public void setPort(int port) {
this.port = port;
}
public String getUser() {
return user;
}
public void setUser(String user) {
this.user = user;
}
public String getPassword() {
return password;
}
public void setPassword(String password) {
this.password = password;
}
}
(4)Cookies:可以用于认证数据设置
1)全局Cookies设置:在创建HttpClient时
/**
* 设置全局Cookies
*
* @param cookies
* @return
*/
public HttpClient setGlobeCookieHttpClient(Map<String, String> cookies) {
HttpClient client = null;
if (cookies != null && cookies.size() > 0) {
BasicCookieStore cookieStore = new BasicCookieStore();
for (Entry<String, String> c : cookies.entrySet()) {
BasicClientCookie cookie = new BasicClientCookie(c.getKey(), c.getValue());
cookie.setDomain("sina.com");
cookie.setPath("/");
cookieStore.addCookie(cookie);
}
client = HttpClientBuilder.create().setDefaultCookieStore(cookieStore).build();
} else {
client = HttpClients.createDefault();
}
return client;
}
2)请求Cookies设置
法一:直接在请求头添加
/**
* 设置Cookie方法1
*
* @param request
* @param cookies
*/
public void setRequestCookiesStr(HttpUriRequest request, Map<String, String> cookies) {
StringBuilder cookieStr = new StringBuilder();
if (cookies != null && cookies.size() > 0) {
for (Entry<String, String> cookie : cookies.entrySet()) {
cookieStr.append(cookie.getKey()).append('=').append(cookie.getValue()).append(';');
}
cookieStr.deleteCharAt(cookieStr.length() - 1);
request.addHeader("Cookie", cookieStr.toString());
}
}
法二:请求是通过context设置
/**
* 设置Cookie 方法2
*
* httpclient.execute(request, context) 方法进行添加
*
* @param cookies
* @return
*/
public HttpContext getetRequestCookieContext(Map<String, String> cookies) {
HttpContext localContext = null;
if (cookies != null && cookies.size() > 0) {
BasicCookieStore cookieStore = new BasicCookieStore();
for (Entry<String, String> c : cookies.entrySet()) {
BasicClientCookie cookie = new BasicClientCookie(c.getKey(), c.getValue());
cookie.setDomain("sina.com");
cookie.setPath("/");
cookieStore.addCookie(cookie);
}
localContext = new BasicHttpContext();
localContext.setAttribute(HttpClientContext.COOKIE_STORE, cookieStore);
}
return localContext;
}
3)读取响应的Cookies
/**
* 响应中获取cookie
*
* @param response
*/
public void getResponseCookies(HttpResponse response) {
//法1
HeaderIterator headerIter = response.headerIterator("Set-Cookie");
while (headerIter.hasNext()) {
Object obj = headerIter.next();
if (obj != null) {
String cookie = ((Header) headerIter.next()).getValue();
String cookieRecord = cookie.substring(0, cookie.indexOf(';'));
System.out.println(cookieRecord);
}
}
//法2
HeaderElementIterator headerElementIter = new BasicHeaderElementIterator(response.headerIterator("Set-Cookie"));
while (headerElementIter.hasNext()) {
HeaderElement elem = headerElementIter.nextElement();
System.out.println("Element : " + elem.getName() + " = " + elem.getValue());
NameValuePair[] params = elem.getParameters();
for (int i = 0; i < params.length; i++) {
System.out.println(" Params : " + params[i].getName() + " | " + params[i].getValue());
}
}
}
附录:完整代码
package x.http.simple.httpclient;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import org.apache.http.Header;
import org.apache.http.HeaderElement;
import org.apache.http.HeaderElementIterator;
import org.apache.http.HeaderIterator;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.NameValuePair;
import org.apache.http.StatusLine;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.HttpClient;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpRequestBase;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.cookie.BasicClientCookie;
import org.apache.http.message.BasicHeaderElementIterator;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.protocol.BasicHttpContext;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
import x.http.simple.HttpProxy;
public class HttpClientDemo {
private CloseableHttpClient httpclient;
public HttpClientDemo() {
this(null);
}
/**
* 代理方式
*
* @param proxy
*/
public HttpClientDemo(HttpProxy proxy) {
if (proxy != null) {
httpclient = getHttpClientWithProxy(proxy);
} else {
httpclient = HttpClients.createDefault();
}
}
// 获得设置代理的httpClent实例
private CloseableHttpClient getHttpClientWithProxy(HttpProxy proxy) {
HttpHost proxyHost = new HttpHost(proxy.getHost(), proxy.getPort());
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(new AuthScope(proxy.getHost(), proxy.getPort()),
new UsernamePasswordCredentials(proxy.getUser(), proxy.getPassword()));
return HttpClients.custom().setProxy(proxyHost).setDefaultCredentialsProvider(credentialsProvider).build();
}
/**
* 发送get请求
*
* @param url
* @throws Exception
*/
public byte[] sendGetRequest(String url) throws Exception {
HttpGet httpGet = new HttpGet(url);
System.out.println("Send Get Request : " + url);
return sendRequest(httpGet);
}
/**
* 发送post 请求
*
* @param request
* @throws Exception
*/
public byte[] sendPostRequest(String url, Map<String, String> params) throws Exception {
HttpPost httpPost = new HttpPost(url);
// 设置参数
if (params != null && params.size() > 0) {
List<NameValuePair> nvps = new ArrayList<NameValuePair>();
for (Entry<String, String> param : params.entrySet()) {
nvps.add(new BasicNameValuePair(param.getKey(), param.getValue()));
}
httpPost.setEntity(new UrlEncodedFormEntity(nvps));
}
System.out.println("Send Post Request : " + url + " , " + params);
return sendRequest(httpPost);
}
// 底层请求打印页面
private byte[] sendRequest(HttpRequestBase request) throws Exception {
CloseableHttpResponse response = httpclient.execute(request);
// 设置超时
setTimeOut(request, 5000);
// 获取返回的状态列表
StatusLine statusLine = response.getStatusLine();
System.out.println("StatusLine : " + statusLine);
if (statusLine.getStatusCode() == HttpStatus.SC_OK) {
try {
// 获取response entity
HttpEntity entity = response.getEntity();
System.out.println("getContentType : " + entity.getContentType().getValue());
System.out.println("getContentEncoding : " + entity.getContentEncoding().getValue());
// 读取响应内容
byte[] responseBody = EntityUtils.toByteArray(entity);
// 关闭响应流
EntityUtils.consume(entity);
return responseBody;
} finally {
response.close();
}
}
return null;
}
// 设置超时重定向
private void setTimeOut(HttpRequestBase request, int timeout) {
RequestConfig requestConfig = RequestConfig.custom().setConnectionRequestTimeout(timeout)
.setConnectTimeout(timeout).setSocketTimeout(timeout).setRedirectsEnabled(true).build();
request.setConfig(requestConfig);
}
/**
* 设置全局Cookies
*
* @param cookies
* @return
*/
public HttpClient setGlobeCookieHttpClient(Map<String, String> cookies) {
HttpClient client = null;
if (cookies != null && cookies.size() > 0) {
BasicCookieStore cookieStore = new BasicCookieStore();
for (Entry<String, String> c : cookies.entrySet()) {
BasicClientCookie cookie = new BasicClientCookie(c.getKey(), c.getValue());
cookie.setDomain("sina.com");
cookie.setPath("/");
cookieStore.addCookie(cookie);
}
client = HttpClientBuilder.create().setDefaultCookieStore(cookieStore).build();
} else {
client = HttpClients.createDefault();
}
return client;
}
/**
* 设置Cookie方法1
*
* @param request
* @param cookies
*/
public void setRequestCookiesStr(HttpUriRequest request, Map<String, String> cookies) {
StringBuilder cookieStr = new StringBuilder();
if (cookies != null && cookies.size() > 0) {
for (Entry<String, String> cookie : cookies.entrySet()) {
cookieStr.append(cookie.getKey()).append('=').append(cookie.getValue()).append(';');
}
cookieStr.deleteCharAt(cookieStr.length() - 1);
request.addHeader("Cookie", cookieStr.toString());
}
}
/**
* 设置Cookie 方法2
*
* httpclient.execute(request, context) 方法进行添加
*
* @param cookies
* @return
*/
public HttpContext getetRequestCookieContext(Map<String, String> cookies) {
HttpContext localContext = null;
if (cookies != null && cookies.size() > 0) {
BasicCookieStore cookieStore = new BasicCookieStore();
for (Entry<String, String> c : cookies.entrySet()) {
BasicClientCookie cookie = new BasicClientCookie(c.getKey(), c.getValue());
cookie.setDomain("sina.com");
cookie.setPath("/");
cookieStore.addCookie(cookie);
}
localContext = new BasicHttpContext();
localContext.setAttribute(HttpClientContext.COOKIE_STORE, cookieStore);
}
return localContext;
}
/**
* 响应中获取cookie
*
* @param response
*/
public void getResponseCookies(HttpResponse response) {
// 法1
HeaderIterator headerIter = response.headerIterator("Set-Cookie");
while (headerIter.hasNext()) {
Object obj = headerIter.next();
if (obj != null) {
String cookie = ((Header) headerIter.next()).getValue();
String cookieRecord = cookie.substring(0, cookie.indexOf(';'));
System.out.println(cookieRecord);
}
}
// 法2
HeaderElementIterator headerElementIter = new BasicHeaderElementIterator(response.headerIterator("Set-Cookie"));
while (headerElementIter.hasNext()) {
HeaderElement elem = headerElementIter.nextElement();
System.out.println("Element : " + elem.getName() + " = " + elem.getValue());
NameValuePair[] params = elem.getParameters();
for (int i = 0; i < params.length; i++) {
System.out.println(" Params : " + params[i].getName() + " | " + params[i].getValue());
}
}
}
public void shutdown() throws Exception {
httpclient.close();
}
public static void main(String[] args) throws Exception {
String url = "http://www.google.com.hk";
HttpClientDemo httpclient = new HttpClientDemo();
byte[] getResponse = httpclient.sendGetRequest(url);
System.out.println("Get Body : " + new String(getResponse, "utf-8"));
byte[] postResponse = httpclient.sendPostRequest(url, null);
System.out.println("Get Body : " + new String(postResponse, "utf-8"));
httpclient.shutdown();
}
}