httpClient使用
前言:httpClient有两个开源项目,使用一定的差异。
httpClient可以用来做什么?
HttpClientk可以用来请求接口,效率比webService的要高;可以用来模拟浏览器操作,如360抢票。
1.第一个版本:
demo:
代码:
package demo;
import java.io.UnsupportedEncodingException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.http.NameValuePair;
import org.apache.http.client.HttpClient;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.params.ClientPNames;
import org.apache.http.client.params.CookiePolicy;
import org.apache.http.impl.client.AbstractHttpClient;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.params.CoreConnectionPNames;
public class HttpClientUtil {
/**
* 获取HttpClient
* @return HttpClient
*/
public static HttpClient getHttpClient() {
// TODO Auto-generated method stub
//线程安全的
HttpClient httpClient = new DefaultHttpClient(new ThreadSafeClientConnManager());
//设置默认策略(设置请求重试处理,请求三次)
DefaultHttpRequestRetryHandler handler = new DefaultHttpRequestRetryHandler(1,false);
((AbstractHttpClient) httpClient).setHttpRequestRetryHandler(handler);
httpClient.getParams().setParameter(CoreConnectionPNames.CONNECTION_TIMEOUT,500);//连接时间
httpClient.getParams().setParameter(CoreConnectionPNames.SO_TIMEOUT,500);//数据传输时间
return httpClient;
}
/**
* 获取HttpGet
* @return
*/
public static HttpGet getHttpGet(String url,Map<String,String> param){
HttpGet httpGet = new HttpGet();
//模拟浏览器请求头,防止被屏蔽
httpGet.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;");
httpGet.setHeader("Accept-Language", "zh-cn");
httpGet.setHeader("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.0.3) Gecko/2008092417 Firefox/3.0.3");
httpGet.setHeader("Accept-Charset", "utf-8");
httpGet.setHeader("Keep-Alive", "300");
httpGet.setHeader("Connection", "Keep-Alive");
httpGet.setHeader("Cache-Control", "no-cache");
//设置编码格式
httpGet.setHeader("Content-Type","application/x-www-form-urlencoded;charset=utf-8");
//设置 HttpClient 接收 Cookie,用与浏览器一样的策略
httpGet.getParams().setParameter(
ClientPNames.COOKIE_POLICY, CookiePolicy.BROWSER_COMPATIBILITY);
StringBuffer urlBuffer = new StringBuffer(url);
if(param != null && param.size() > 0){
if(param != null && param.size() > 0){
Set<String> set = param.keySet();
int i = 0;
for(String key : set){
if(i == 0){
urlBuffer.append("?");
}else{
urlBuffer.append("&");
}
urlBuffer.append(key + "=" + param.get(key));
i++;
}
}
}
//设置链接
try {
URI uri = new URI(urlBuffer.toString());
httpGet.setURI(uri);
} catch (URISyntaxException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return httpGet;
}
/**
* 获取HttpPost
* @return HttpPost
*/
public static HttpPost getHttpPost(String url,Map<String,String> param){
HttpPost httpPost = new HttpPost(url);
//模拟浏览器请求头,防止被屏蔽
httpPost.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;");
httpPost.setHeader("Accept-Language", "zh-cn");
httpPost.setHeader("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.0.3) Gecko/2008092417 Firefox/3.0.3");
httpPost.setHeader("Accept-Charset", "utf-8");
httpPost.setHeader("Keep-Alive", "300");
httpPost.setHeader("Connection", "Keep-Alive");
httpPost.setHeader("Cache-Control", "no-cache");
//设置 HttpClient 接收 Cookie,用与浏览器一样的策略
httpPost.getParams().setParameter(
ClientPNames.COOKIE_POLICY, CookiePolicy.BROWSER_COMPATIBILITY);
//设置编码格式
httpPost.setHeader("Content-Type","application/x-www-form-urlencoded;charset=utf-8");
List<NameValuePair> formparams = new ArrayList<NameValuePair>();
if(param != null && param.size() > 0){
Set<String> set = param.keySet();
for(String key : set){
formparams.add(new BasicNameValuePair(key, param.get(key)));
}
}
UrlEncodedFormEntity entity = null;
try {
entity = new UrlEncodedFormEntity(formparams,"utf-8");
httpPost.setEntity(entity);
} catch (UnsupportedEncodingException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return httpPost;
}
}
测试:
package demo;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.params.CoreProtocolPNames;
import org.apache.http.util.EntityUtils;
public class HttpClientUtilTest {
public static void toTestMethod(HttpClient client,HttpUriRequest request){
try {
HttpResponse response = client.execute(request);
if(response.getStatusLine().getStatusCode() == HttpStatus.SC_OK){
//获取response的编码
System.out.println(response.getParams().getParameter(CoreProtocolPNames.HTTP_CONTENT_CHARSET));
//获取响应内容(流)
// InputStream in = response.getEntity().getContent();
// EntityUtils.toString(Entity, "utf-8"); //String
// EntityUtils.toByteArray(Entity) //Byte[]
HttpEntity entity = response.getEntity();
String str = EntityUtils.toString(entity,"GBK");
System.out.println(str);
}else{
System.out.println("响应失败!");
}
} catch (ClientProtocolException e) {
System.out.println("连接错误!");
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
System.out.println("连接错误!");
// TODO Auto-generated catch block
e.printStackTrace();
}finally{
//关闭连接,释放资源
client.getConnectionManager().shutdown();
}
}
/**
* @param args
*/
public static void main(String[] args) {
// TODO Auto-generated method stub
//地址
String url = "http://www.ip138.com:8080/search.asp";
//链接地址为 ?action=mobile&mobile=1868478044
Map<String,String> param = new HashMap<String, String>();
param.put("action", "mobile");
param.put("mobile", "1868478044");
HttpClient client = HttpClientUtil.getHttpClient();
//测试Get
HttpGet httpGet = HttpClientUtil.getHttpGet(url, param);
toTestMethod(client, httpGet);
client = HttpClientUtil.getHttpClient();
//测试Post
HttpPost httpPost = HttpClientUtil.getHttpPost(url, param);
toTestMethod(client, httpPost);
}
}
2.另一个版本
Demo:
代码:
package demo;
import java.util.Map;
import java.util.Set;
import org.apache.commons.httpclient.DefaultHttpMethodRetryHandler;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
import org.apache.commons.httpclient.NameValuePair;
import org.apache.commons.httpclient.URI;
import org.apache.commons.httpclient.URIException;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.httpclient.params.HttpMethodParams;
import org.apache.http.params.CoreConnectionPNames;
/**
* 测试HttpClient调用
*
*/
public class HttpClientUtil{
/**
* 获取Client
*/
public static HttpClient getHttpClient(){
//创建线程安全的Http客户端
MultiThreadedHttpConnectionManager connectionManager =
new MultiThreadedHttpConnectionManager();
HttpClient client = new HttpClient(connectionManager);
//设置时间
client.getParams().setParameter(CoreConnectionPNames.CONNECTION_TIMEOUT,2000);//连接时间(毫秒)
client.getParams().setParameter(CoreConnectionPNames.SO_TIMEOUT,2000);//数据传输时间(毫秒)
return client;
}
/**
* 获得Post
* @param param 参数
* @param url 请求地址
*/
public static PostMethod getPostMethod(String url,Map<String,String> param){
System.out.println("Post提交");
PostMethod postMethod = new PostMethod(url);
//模拟浏览器请求头,防止被屏蔽
postMethod.setRequestHeader("Accept", "text/html,application/xhtml+xml,application/xml;");
postMethod.setRequestHeader("Accept-Language", "zh-cn");
postMethod.setRequestHeader("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.0.3) Gecko/2008092417 Firefox/3.0.3");
postMethod.setRequestHeader("Accept-Charset", "utf-8");
postMethod.setRequestHeader("Keep-Alive", "300");
postMethod.setRequestHeader("Connection", "Keep-Alive");
postMethod.setRequestHeader("Cache-Control", "no-cache");
//设置字符编码
postMethod.setRequestHeader("Content-Type","application/x-www-form-urlencoded;charset=utf-8");
//使用系统提供的默认的恢复策略 (设置请求重试处理,请求三次)
postMethod.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
new DefaultHttpMethodRetryHandler());
// 填入各个表单域的值
if(param != null && param.size() > 0){
NameValuePair[] datas = new NameValuePair[param.size()];
Set<String> set = param.keySet();
int i = 0;
for(String key : set){
NameValuePair date = new NameValuePair(key,param.get(key));
datas[i] = date;
i++;
}
postMethod.setRequestBody(datas);
}
return postMethod;
}
/**
* 获得get
* @param param 参数
* @param url 请求地址
*/
public static GetMethod getGetMethod(String url,Map<String,String> param){
GetMethod getMethod = new GetMethod();
//模拟浏览器请求头,防止被屏蔽
getMethod.setRequestHeader("Accept", "text/html,application/xhtml+xml,application/xml;");
getMethod.setRequestHeader("Accept-Language", "zh-cn");
getMethod.setRequestHeader("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.0.3) Gecko/2008092417 Firefox/3.0.3");
getMethod.setRequestHeader("Accept-Charset", "utf-8");
getMethod.setRequestHeader("Keep-Alive", "300");
getMethod.setRequestHeader("Connection", "Keep-Alive");
getMethod.setRequestHeader("Cache-Control", "no-cache");
//使用系统提供的默认的恢复策略 (设置请求重试处理,请求三次)
getMethod.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
new DefaultHttpMethodRetryHandler());
// 填入各个链接的请求参数
StringBuffer urlBuffer = new StringBuffer(url);
if(param != null && param.size() > 0){
Set<String> set = param.keySet();
int i = 0;
for(String key : set){
if(i == 0){
urlBuffer.append("?");
}else{
urlBuffer.append("&");
}
urlBuffer.append(key + "=" + param.get(key));
i++;
}
}
//设置链接
try {
URI uri = new URI(urlBuffer.toString(), false, "utf-8");
getMethod.setURI(uri);
} catch (URIException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return getMethod;
}
}
测试:
package demo;
import java.io.IOException;
public class HttpClientUtilTest{
/**
* 测试
* @param client
* @param method
*/
public static void toTestMethod(HttpClient client,HttpMethod method) {
try{
//执行 获取状态码
int statusCode = client.executeMethod(method);
//判读是否成功响应
if(statusCode == HttpStatus.SC_OK){
/* 获取返回参数
* getMethod.getResponseBodyAsStream();
getMethod.getResponseBodyAsString();*/
System.out.println(method.getParams().getParameter(CoreProtocolPNames.HTTP_CONTENT_CHARSET));
//转码
String result = new String(method.getResponseBody(),"gbk");
System.out.println("response:" + result);
}else{
System.out.println("响应失败!");
}
} catch (UnsupportedEncodingException e) {
// TODO Auto-generated catch block
System.out.println("连接错误!");
e.printStackTrace();
} catch (HttpException e) {
// TODO Auto-generated catch block
System.out.println("连接错误!");
e.printStackTrace();
} catch (IOException e) {
System.out.println("连接错误!");
// TODO Auto-generated catch block
e.printStackTrace();
}finally{
//关闭连接
method.releaseConnection();
}
}
public static void main(String[] args) {
//地址
String url = "http://www.ip138.com:8080/search.asp";
//链接地址为 ?action=mobile&mobile=1868478044
Map<String,String> param = new HashMap<String, String>();
param.put("action", "mobile");
param.put("mobile", "1868478044");
HttpClient client = HttpClientUtil.getHttpClient();
//测试Get
GetMethod getMethod = HttpClientUtil.getGetMethod(url, param);
toTestMethod(client, getMethod);
//测试Post
PostMethod postMethod = HttpClientUtil.getPostMethod(url, param);
toTestMethod(client, postMethod);
}
}
2.HttpClient管理Cookie,保持登陆状态
1.第一个版本:
package loginTest;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import org.apache.http.Header;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.params.HttpParams;
import org.apache.http.util.EntityUtils;
import org.omg.Dynamic.Parameter;
import demo.HttpClientUtil;
public class LoginTest {
public static String loginUrl = "http://www.51zxw.net/bbs/login.asp?action=chk";
public static String myUrl = "http://www.51zxw.net/bbs/usermanager.asp";
public static Header[] headers = null;
public static HttpClient client;
/**
* 登陆
*/
public static void login(){
client = HttpClientUtil.getHttpClient();
Map<String,String> param = new HashMap<String,String>();
param.put("username", "m474511700");
param.put("password", "474511700");
HttpPost method = HttpClientUtil.getHttpPost(loginUrl, param);
try {
HttpResponse response = client.execute(method);
if(response.getStatusLine().getStatusCode() == HttpStatus.SC_OK){
System.out.println(EntityUtils.toString(response.getEntity(), "gbk"));
//获取Cookie值
headers = response.getHeaders("Set-Cookie");
for(Header header : headers){
System.out.println(header.getName() + "___" + header.getValue());
}
}else{
System.out.println("连接发生错误!");
}
} catch (ClientProtocolException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
/**
* 去我的会员中心
*/
public static void goMy(){
HttpGet method = HttpClientUtil.getHttpGet(myUrl,null);
/**
* 如果使用同一个HttpClient对象,则HttpClient会自动管理Cookie信息,保持登陆状态
* 如果不是同一个HttpClient对象,则需要手动添加Cookie信息,维持登陆状态
* 不是同一个HttpClient对象的处理:
*
StringBuffer sb = new StringBuffer();
client = HttpClientUtil.getHttpClient();
if(headers != null && headers.length > 0){
for(Header header: headers) {
String value = header.getValue();
int pos = value.indexOf(";");
sb.append(value.substring(0, pos));
sb.append(";");
}
}
method.addHeader("Cookie",sb.toString());
//打印Cookie
System.out.println(method.getHeaders("Cookie")[0].getValue());
*
*
*/
try {
HttpResponse response = client.execute(method);
if(response.getStatusLine().getStatusCode() == HttpStatus.SC_OK){
System.out.println(EntityUtils.toString(response.getEntity(), "gbk"));
}else{
System.out.println("连接发生错误!");
}
} catch (ClientProtocolException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
public static void main(String[] args) {
login();
goMy();
}
}
2.另一个版本
package login;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.PostMethod;
import demo.HttpClientUtil;
public class LoginTest {
public static String loginUrl = "http://www.51zxw.net/bbs/login.asp?action=chk";//登陆
public static String myUrl = "http://www.51zxw.net/bbs/usermanager.asp"; //个人中心
public static Header[] headers = null;
public static void main(String[] args) {
HttpClient client = HttpClientUtil.getHttpClient();
Map<String,String> param = new HashMap<String,String>();
param.put("username", "m474511700");
param.put("password", "474511700");
PostMethod method = HttpClientUtil.getPostMethod(loginUrl, param);
try {
int status = client.executeMethod(method);
if(status == HttpStatus.SC_OK){ //登陆成功
//保存Cookie信息
headers = method.getResponseHeaders("Set-Cookie");
//method.getResponseBody() 不建议使用,这里只是输出测试
System.out.println(new String(method.getResponseBody(),"GBK"));
//链接至个人中心
GetMethod getMethod = HttpClientUtil.getGetMethod(myUrl, null);
/**
* 如果使用同一个HttpClient对象,则HttpClient会自动管理Cookie信息,保持登陆状态
* 如果不是同一个HttpClient对象,则需要手动添加Cookie信息,维持登陆状态
* 不是同一个HttpClient对象的处理:
*
StringBuffer sb = new StringBuffer();
client = HttpClientUtil.getHttpClient();
if(headers != null && headers.length > 0){
for(Header header: headers) {
String value = header.getValue();
System.out.println(value);
int pos = value.indexOf(";");
sb.append(value.substring(0, pos));
sb.append(";");
}
}
//给请求添加Cookie
getMethod.addRequestHeader("Cookie",sb.toString());
*/
int myStatus = client.executeMethod(getMethod);
if(myStatus == HttpStatus.SC_OK){
//method.getResponseBody() 不建议使用,这里只是输出测试
System.out.println(new String(getMethod.getResponseBody(),"GBK"));
}
}
} catch (HttpException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
有些网站的请求地址是以Https开头的,http我们知道是一种传输协议,那https是什么呢?
HTTPS(全称:Hyper Text Transfer Protocol over Secure Socket Layer),是以安全为目标的
HTTP通道,简单讲是HTTP的安全版。即HTTP下加入SSL层,HTTPS的安全基础是SSL,因此加密的详细内容就需要SSL。
请求这些网站都需要证书验证,证书验证又分两种,一种是服务器验证客户端的证书,我们称为单项验证(如12306);另一种是服务器验证后,会将自己的证书发送过来给可客户端验证是否是自己所信任的证书机构颁发的,如果是就继续执行,如果不是客户浏览器就给客户一个警告消息:警告客户这个证书不是可以信赖的,询问客户是否需要继续。
3.服务器单向验证证书
1.到浏览器中下载证书
导出以后,会在你填写的文件目录下看到一个这样的文件
2.生成keystore文件
利用java中的Keytool证书管理工具
命令创建keystore,在cmd运行:
keytool -import -alias myKey -keystore C:\Users\Administrator\Desktop\myKey.keystore -file C:\Users\Administrator\Desktop\myKey.cer
myKey 是别名
C:\Users\Administrator\Desktop\myKey.keystore 生成的路径
C:\Users\Administrator\Desktop\myKey.cer 下载证书的路径
设置密码后(密码一般默认设置为"changeit"),添加信任输入y.
生成keystore文件后,引入工程中。
3.调用证书代码:
package ssl;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.security.KeyManagementException;
import java.security.KeyStore;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.UnrecoverableKeyException;
import java.security.cert.CertificateException;
import org.apache.commons.io.IOUtils;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.conn.scheme.Scheme;
import org.apache.http.conn.ssl.SSLSocketFactory;
import org.apache.http.impl.client.AbstractHttpClient;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager;
import org.apache.http.params.CoreConnectionPNames;
import org.apache.http.params.CoreProtocolPNames;
/**
* SSL单项认证
* @author Administrator
*
*/
public class SSLTest {
@SuppressWarnings("deprecation")
public static void downloadImage() throws NoSuchAlgorithmException, CertificateException, IOException, KeyStoreException, KeyManagementException, UnrecoverableKeyException {
//获得密匙库
String keyPath = "key/myKey.keystore";
//下载验证码
String url = "https://kyfw.12306.cn/otn/passcodeNew/getPassCodeNew?module=login&rand=sjrand";
KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType());
FileInputStream instream = new FileInputStream(new File(keyPath));
//密匙库的密码
trustStore.load(instream, "changeit".toCharArray());
//注册密匙库
SSLSocketFactory socketFactory = new SSLSocketFactory(trustStore);
//不校验域名
socketFactory.setHostnameVerifier(SSLSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
Scheme sch = new Scheme("https", 443, socketFactory);
//线程安全的
HttpClient httpClient = new DefaultHttpClient(new ThreadSafeClientConnManager());
//加入签名
httpClient.getConnectionManager().getSchemeRegistry().register(sch);
HttpGet getMethod = new HttpGet(url);
//设置策略(设置请求重试处理,请求1次)
DefaultHttpRequestRetryHandler handler = new DefaultHttpRequestRetryHandler(1,false);
((AbstractHttpClient) httpClient).setHttpRequestRetryHandler(handler);
httpClient.getParams().setParameter(CoreConnectionPNames.CONNECTION_TIMEOUT,500);//连接时间
httpClient.getParams().setParameter(CoreConnectionPNames.SO_TIMEOUT,500);//数据传输时间
getMethod.setHeader("Content-Type","application/x-www-form-urlencoded;charset=utf-8");
// HttpPost httpPost = new HttpPost(url);
// StringEntity entity = new StringEntity("");
// httpPost.setEntity(entity);
//生成图片
try {
HttpResponse resp = httpClient.execute(getMethod);
System.out.println(resp.getStatusLine().getStatusCode());
if (resp.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
//生成图片
String picName = "img\\test.jpg";
InputStream inputStream = resp.getEntity().getContent();
//获取response的编码
System.out.println(resp.getParams().getParameter(CoreProtocolPNames.HTTP_CONTENT_CHARSET));
OutputStream outStream = new FileOutputStream(picName);
IOUtils.copy(inputStream, outStream);
outStream.close();
}else{
System.out.println("失败!");
}
} catch (Exception e) {
e.printStackTrace();
} finally {
//关闭连接,释放资源
httpClient.getConnectionManager().shutdown();
}
}
/**
* @param args
* @throws Exception
*/
public static void main(String[] args) throws Exception {
downloadImage();
}
}
本文仅限交流学习使用。