1. HttpClient简介
官网地址： http://hc.apache.org/
- HttpClient 是Apache Jakarta Common 下的子项目，可以用来提供高效的、最新的、功能丰富的支持 HTTP 协议的客户端编程工具包，并且它支持 HTTP 协议最新的版本和建议。
- 实现了所有 HTTP 的方法，支持自动转向，支持 HTTPS 协议，支持代理服务器等
2. 创建一个MavenWeb工程
2.1 引入jar包，新版有优化，我这新的是v4.5.10，旧的是v4.1.2
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.zmj</groupId>
<artifactId>HttpClientDemo</artifactId>
<packaging>war</packaging>
<version>0.0.1-SNAPSHOT</version>
<name>HttpClientDemo Maven Webapp</name>
<url>http://maven.apache.org</url>
<properties> <!--项目开发者属性-->
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<!-- 文件拷贝时的编码，设置为UTF-8 -->
<spring.version>4.0.0.RELEASE</spring.version> <!-- 为了确定每个框架的版本号，锁定版本 -->
<jackson.version>2.9.8</jackson.version>
</properties>
<dependencies>
<!-- SpringMVC的一些相关jar -->
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-context</artifactId>
<version>${spring.version}</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-beans</artifactId>
<version>${spring.version}</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-webmvc</artifactId>
<version>${spring.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>${jackson.version}</version>
</dependency>
<!-- HttpClient 用到的jar -->
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.10</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.5</version>
</dependency>
</dependencies>
<build>
<finalName>HttpClientDemo</finalName>
<pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.tomcat.maven</groupId>
<artifactId>tomcat8-maven-plugin</artifactId>
<version>8.0.38</version>
<configuration>
<path>/</path>
<port>8080</port>
</configuration>
</plugin>
</plugins>
</pluginManagement>
</build>
</project>2.2 新建工具类v4.1.2
package com.zmj.utils;
import java.util.List;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.util.EntityUtils;
public class MyHttpClientOld {//基于httpclient4.1.2版本的
/**
*
* @param url 请求路径
* @param encode 字符编码 比如 org.apache.http.protocol.HTTP.UTF_8
* @return
*/
public static String doGetStr(String url,String encode) {
DefaultHttpClient hc = new DefaultHttpClient();
String result = null;
try {
HttpGet httpGet = new HttpGet(url);
HttpResponse response = hc.execute(httpGet);
HttpEntity entity = response.getEntity();
if(entity != null) {
result = EntityUtils.toString(entity,encode);
}
} catch (Exception e) {
e.printStackTrace();
} finally {
hc.getConnectionManager().shutdown();
}
return result;
}
/**
*
* @param url 请求路径
* @param params 请求参数 params.add(new BasicNameValuePair("key","value"))
* @param encode 字符编码 比如 org.apache.http.protocol.HTTP.UTF_8
* @return
*/
public static String doPostStr(String url,List<NameValuePair> params,String encode) {
DefaultHttpClient hc=new DefaultHttpClient();
try {
HttpPost httppost=new HttpPost(url);
httppost.setEntity(new UrlEncodedFormEntity(params,encode));
HttpResponse resp=hc.execute(httppost);
HttpEntity entity=resp.getEntity();
if (entity != null) {
return EntityUtils.toString(entity);
}
} catch (Exception e) {
e.printStackTrace();
} finally {
hc.getConnectionManager().shutdown();
}
return null;
}
/**
*
* @param url 请求路径
* @param paramStr 参数字符串
* @param encode 字符编码 比如 org.apache.http.protocol.HTTP.UTF_8
* @return
*/
public static String doPostStr(String url,String paramStr,String encode) {
DefaultHttpClient httpClient = new DefaultHttpClient();
HttpPost httpPost = new HttpPost(url);
try {
httpPost.setEntity(new StringEntity(paramStr,encode));
HttpResponse response = httpClient.execute(httpPost);
return EntityUtils.toString(response.getEntity(),encode);
} catch (Exception e) {
e.printStackTrace();
}
return null;
}
}2.3 新建工具类v4.5.10
package com.zmj.utils;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.List;
import java.util.Map;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.NameValuePair;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.config.RequestConfig.Builder;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
public class MyHttpClient {//基于httpclient4.5.10版本的
/**
* @param connectTimeout 设置连接时间 毫秒
* @param socketTimeout 设置读取时间 毫秒
* @param proxy
* @return
*/
public static RequestConfig getRequestConfig(Integer connectTimeout,Integer socketTimeout,HttpHost proxy) {
Builder custom = RequestConfig.custom();
if (connectTimeout != null && connectTimeout > 0) {
custom = custom.setConnectTimeout(connectTimeout);
}
if (socketTimeout != null && socketTimeout > 0) {
custom = custom.setSocketTimeout(socketTimeout);
}
if (proxy != null) {
custom = custom.setProxy(proxy);
}
return custom.build();
}
public static HttpGet getHttpGet(String url,RequestConfig config,Map<String,String> paramMap,Map<String,String> headerMap) {
HttpGet httpGet = null;
if (paramMap != null) {
try {
URIBuilder uriBuilder = new URIBuilder(url);
for(Map.Entry<String, String> entry : paramMap.entrySet()) {
uriBuilder.addParameter(entry.getKey(), entry.getValue());
}
URI uri = uriBuilder.build();
httpGet = new HttpGet(uri);
} catch (URISyntaxException e1) {
e1.printStackTrace();
}
} else {
httpGet = new HttpGet(url);
}
if (config != null) {
httpGet.setConfig(config);
}
if (headerMap != null) {
for(Map.Entry<String, String> entry : headerMap.entrySet()) {
httpGet.setHeader(entry.getKey(),entry.getValue());
}
}
return httpGet;
}
public static HttpGet getHttpGet(String url) {
return getHttpGet(url, null, null, null);
}
public static HttpGet getHttpGet(String url,RequestConfig config) {
return getHttpGet(url, config, null, null);
}
/**
*
* @param url 请求路径
* @param encode 字符编码 比如 StandardCharsets.UTF_8
* @return
*/
public static String doGetStr(String url,String encode,HttpGet httpGet) {
CloseableHttpClient hc = HttpClients.createDefault();
CloseableHttpResponse response;
String result = null;
try {
response = hc.execute(httpGet);
if (response.getStatusLine().getStatusCode() == 200) {
HttpEntity entity = response.getEntity();
if(entity != null) {
result = EntityUtils.toString(entity,encode);
}
}
} catch (ClientProtocolException e1) {
e1.printStackTrace();
} catch (IOException e1) {
e1.printStackTrace();
} finally {
close(hc);
}
return result;
}
/**
*
* @param url 请求路径
* @param params 请求参数 params.add(new BasicNameValuePair("key","value"))
* @param encode 字符编码 比如 org.apache.http.protocol.HTTP.UTF_8
* @param connectTimeout 设置连接时间 毫秒
* @param socketTimeout 设置读取时间 毫秒
* @return
*/
public static String doPostStr(String url,List<NameValuePair> params,String encode,int connectTimeout,int socketTimeout,HttpHost proxy) {
CloseableHttpClient hc = HttpClients.createDefault();
try {
HttpPost httppost=new HttpPost(url);
httppost.setEntity(new UrlEncodedFormEntity(params,encode));
CloseableHttpResponse resp=hc.execute(httppost);
HttpEntity entity=resp.getEntity();
if (entity != null) {
return EntityUtils.toString(entity);
}
} catch (Exception e) {
e.printStackTrace();
} finally {
close(hc);
}
return null;
}
/**
*
* @param url 请求路径
* @param paramStr 参数字符串
* @param encode 字符编码 比如 org.apache.http.protocol.HTTP.UTF_8
* @param connectTimeout 设置连接时间 毫秒
* @param socketTimeout 设置读取时间 毫秒
* @return
*/
public static String doPostStr(String url,String paramStr,String encode,int connectTimeout,int socketTimeout,HttpHost proxy) {
CloseableHttpClient hc = HttpClients.createDefault();
try {
HttpPost httpPost = new HttpPost(url);
httpPost.setEntity(new StringEntity(paramStr,encode));
CloseableHttpResponse response = hc.execute(httpPost);
return EntityUtils.toString(response.getEntity(),encode);
} catch (Exception e) {
e.printStackTrace();
} finally {
close(hc);
}
return null;
}
public static void close(CloseableHttpClient hc) {
try {
if (hc != null) hc.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}2.4 TestHttpGet
package com.zmj.test;
import java.io.IOException;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import org.apache.http.HttpEntity;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.junit.Test;
import com.zmj.utils.MyHttpClient;
public class TestHttpGet {
@Test
public void testGetNoParam() {
CloseableHttpClient hc = HttpClients.createDefault();
HttpGet httpGet = new HttpGet("http://www.baidu.com");
CloseableHttpResponse response;
try {
response = hc.execute(httpGet);
HttpEntity entity = response.getEntity();
String result = EntityUtils.toString(entity,StandardCharsets.UTF_8);
System.out.println("网页内容是：" + result);
} catch (ClientProtocolException e1) {
e1.printStackTrace();
} catch (IOException e1) {
e1.printStackTrace();
} finally {
MyHttpClient.close(hc);
}
}
@Test
public void testGetHasParam() {
CloseableHttpClient hc = HttpClients.createDefault();
CloseableHttpResponse response;
try {
URIBuilder builder = new URIBuilder("https://www.sogou.com/web");
builder.addParameter("query", "西游记");
HttpGet httpGet = new HttpGet(builder.build());
httpGet.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36");
response = hc.execute(httpGet);
HttpEntity entity = response.getEntity();
String result = EntityUtils.toString(entity,StandardCharsets.UTF_8);
System.out.println("网页内容是：" + result);
} catch (URISyntaxException e) {
e.printStackTrace();
} catch (ClientProtocolException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} finally {
MyHttpClient.close(hc);
}
}
}2.5 TestHttpPost
package com.zmj.web.controller;
import java.util.HashMap;
import java.util.Map;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.ResponseBody;
import com.zmj.pojo.TestUser;
@Controller
@RequestMapping("/myPostDemo")
public class MyPostDemoController {
@RequestMapping(value="/postNoParam",method=RequestMethod.POST)
@ResponseBody
public Object postNoParam() {
Map<String,String> paramMap = new HashMap<>();
paramMap.put("msg", "post is OK");
return paramMap;
}
@RequestMapping(value="/postHasParam",method=RequestMethod.POST,produces=MediaType.APPLICATION_JSON_VALUE+";charset=utf-8")
@ResponseBody
public Object postHasParam(String name,String pwd) {
System.out.println("传参 name=" + name + ",pwd="+pwd);
Map<String,String> paramMap = new HashMap<>();
paramMap.put("name", name);
paramMap.put("pwd", pwd);
return paramMap;
}
@RequestMapping(value="/postWithJson",method=RequestMethod.POST)
@ResponseBody
public Object postWithJson(@RequestBody TestUser user) {
System.out.println("传参 name=" + user.getName() + ",pwd="+user.getPwd());
Map<String,String> paramMap = new HashMap<>();
paramMap.put("name", user.getName());
paramMap.put("pwd", user.getPwd());
return paramMap;
}
}package com.zmj.test;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import org.apache.http.HttpEntity;
import org.apache.http.NameValuePair;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.junit.Test;
import com.zmj.utils.MyHttpClient;
public class TestHttpPost {
@Test
public void testPostNoParam() {
CloseableHttpClient hc = HttpClients.createDefault();
HttpPost httpPost = new HttpPost("http://localhost:8080/HttpClientDemo/myPostDemo/postNoParam");
try {
CloseableHttpResponse response = hc.execute(httpPost);
HttpEntity entity = response.getEntity();
String result = EntityUtils.toString(entity,StandardCharsets.UTF_8);
System.out.println("内容是：" + result);
} catch (ClientProtocolException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} finally {
MyHttpClient.close(hc);
}
}
@Test
public void testPostHasParam() {
CloseableHttpClient hc = HttpClients.createDefault();
HttpPost httpPost = new HttpPost("http://localhost:8080/HttpClientDemo/myPostDemo/postHasParam");
List<NameValuePair> params = new ArrayList<NameValuePair>();
params.add(new BasicNameValuePair("name","张三丰"));
params.add(new BasicNameValuePair("pwd","1q2w3e4r"));
UrlEncodedFormEntity entity = new UrlEncodedFormEntity(params,StandardCharsets.UTF_8);
httpPost.setEntity(entity);
try {
CloseableHttpResponse response = hc.execute(httpPost);
int statusCode = response.getStatusLine().getStatusCode();
System.out.println("状态码="+statusCode);
HttpEntity httpEntity = response.getEntity();
String content = EntityUtils.toString(httpEntity,StandardCharsets.UTF_8);
System.out.println("content="+content);
} catch (ClientProtocolException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} finally {
MyHttpClient.close(hc);
}
}
@Test
public void testPostWithJson() {
CloseableHttpClient hc = HttpClients.createDefault();
HttpPost httpPost = new HttpPost("http://localhost:8080/HttpClientDemo/myPostDemo/postWithJson");
String jsonStr = "{\"name\":\"张三丰\",\"pwd\":\"1q2w3e4r\"}";
StringEntity entity = new StringEntity(jsonStr,ContentType.APPLICATION_JSON);
httpPost.setEntity(entity);
try {
CloseableHttpResponse response = hc.execute(httpPost);
int statusCode = response.getStatusLine().getStatusCode();
System.out.println("状态码="+statusCode);
HttpEntity httpEntity = response.getEntity();
String content = EntityUtils.toString(httpEntity,StandardCharsets.UTF_8);
System.out.println("content="+content);
} catch (ClientProtocolException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} finally {
MyHttpClient.close(hc);
}
}
}2.6 有道翻译
package com.zmj.test;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;
import org.apache.http.NameValuePair;
import org.apache.http.message.BasicNameValuePair;
import org.junit.jupiter.api.Test;
import com.zmj.utils.MyHttpClientOld;
import org.apache.http.protocol.HTTP;
public class HttpClient_YouDaoFanYi {//有道翻译这个是基于httpclient4.1.2版本的
/*
参数说明：
type - 返回结果的类型，固定为data
doctype - 返回结果的数据格式，xml或json或jsonp
version - 版本，当前为1.1
q - 要翻译的文本，必须是UTF-8编码，字符长度不能超过200个字符，get请求有时需要进行urlencode编码
only - 可选参数，dict表示只获取词典数据，translate表示只获取翻译数据，默认为都获取
注： 词典结果只支持中英互译，翻译结果支持英日韩法俄西到中文的翻译以及中文到英语的翻译
errorCode：
0 - 正常
20 - 要翻译的文本过长
30 - 无法进行有效的翻译
40 - 不支持的语言类型
50 - 无效的key
60 - 无词典结果，仅在获取词典结果生效
*/
@Test
public void testDoGet() {//如果是get请求，要注意特殊的字符，比如空格
String queryWord = "测试";
//queryWord = "test";
//queryWord = "How old are you!"; queryWord=URLEncoder.encode(queryWord);//比如，它会把空格变成加号
//queryWord = "你多大了？";
//queryWord = "你 多 大 了？"; queryWord= queryWord.replace(" ", "");//中文不去除空格不会报错，但有道翻译的时候，会翻译错误 How big are you?
String doGetUrl = "http://fanyi.youdao.com/openapi.do?"
//+ "keyfrom=youdianbao&key=1661829537"
+ "keyfrom=wangtuizhijia&key=1048394636"
+ "&type=data&doctype=json&version=1.1&q="+queryWord;
String result = MyHttpClientOld.doGetStr(doGetUrl,HTTP.UTF_8);
System.out.println("doGetStr=" + result);
}
@Test
public void testDoPost() {
String queryWord = "测试";
//queryWord = "test";
//queryWord = "How old are you!";
//queryWord = "你多大了？";
//queryWord = "你 多 大 了？"; queryWord= queryWord.replace(" ", ""); //不去除空格不会报错，但有道翻译的时候，会翻译错误 How big are you?
String doPostUrl = "http://fanyi.youdao.com/openapi.do";
List<NameValuePair> params = new ArrayList<NameValuePair>();
//params.add(new BasicNameValuePair("keyfrom","youdianbao"));
//params.add(new BasicNameValuePair("key","1661829537"));
params.add(new BasicNameValuePair("keyfrom","wangtuizhijia"));
params.add(new BasicNameValuePair("key","1048394636"));
params.add(new BasicNameValuePair("type","data"));
params.add(new BasicNameValuePair("doctype","json"));
params.add(new BasicNameValuePair("version","1.1"));
params.add(new BasicNameValuePair("q",queryWord));
String result = MyHttpClientOld.doPostStr(doPostUrl,params,HTTP.UTF_8);
System.out.println("doPostStr=" + result);
}
}2.7 高匿代理
package com.zmj.test;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import com.zmj.utils.MyHttpClient;
public class HighLevelAnonymousAgent {//高匿代理
public static void main(String[] args) {
CloseableHttpClient hc = HttpClients.createDefault();
HttpGet httpGet = new HttpGet("https://www.tuicool.com/");
//配置代理IP
//有时候某个网站因为爬虫访问过度，而将IP纳入黑名单，
//这时需要代理IP，作一个队列，哪怕封了一个，再来一个即可
HttpHost proxy = new HttpHost("175.155.213.235",9999);
httpGet.setConfig(MyHttpClient.getRequestConfig(10000, 10000, proxy));
//如果不加这句话，会报 系统检测亲不是真人行为，因系统资源限制，我们只能拒绝你的请求
//说明它只允许你通过浏览器请求，那么这里相当于模拟浏览器请求
httpGet.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36");
CloseableHttpResponse response = null;
try {
response = hc.execute(httpGet);
System.out.println("Status:" + response.getStatusLine());
System.out.println("StatusCode:" + response.getStatusLine().getStatusCode());
HttpEntity entity = response.getEntity();
if (entity != null) {
//获取Content-Type
System.out.println(entity.getContentType().getName() + ":" + entity.getContentType().getValue());
}
String result = EntityUtils.toString(entity,StandardCharsets.UTF_8);
System.out.println("网页内容是：" + result);
} catch (ClientProtocolException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} finally {
MyHttpClient.close(hc);
}
}
}2.8 抓取图片
package com.zmj.test;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import org.apache.commons.io.FileUtils;
import org.apache.http.HttpEntity;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import com.zmj.utils.MyHttpClient;
public class FetchImages {
public static void main(String[] args) {
CloseableHttpClient hc = HttpClients.createDefault();
HttpGet httpGet = new HttpGet("http://www.java1234.com/uploads/allimg/170123/1-1F123124629330.jpg");
//设置连接时间为10秒钟 设置读取时间为10秒钟
httpGet.setConfig(MyHttpClient.getRequestConfig(10000, 10000,null));
httpGet.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36");
CloseableHttpResponse response = null;
try {
response = hc.execute(httpGet);
HttpEntity entity = response.getEntity();
if (entity != null) {
InputStream source = entity.getContent();
FileUtils.copyToFile(source, new File("D:/"+System.currentTimeMillis()+".jpg"));
}
} catch (ClientProtocolException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} finally {
MyHttpClient.close(hc);
}
}
}时间有点仓促，还在持续更新中。。。。。。。
597
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
