HttpClient入门简单使用
视频教程:乐之者Java
使用场景: 爬虫,多系统之间接口交互
jdk原生api发送请求
/**
* 使用jdk原生的api来请求网页
*/
@Test
public void test() throws Exception{
//访问地址
String urlStr = "https://www.baidu.com/";
URL url = new URL(urlStr);
//获取url的连接
HttpURLConnection connection = (HttpURLConnection)url.openConnection();
/**
* Http请求格式
* 请求行 空格 请求头 请求体
*/
//设置属性请求类型,编码
connection.setRequestMethod("GET");
connection.setRequestProperty("Accept-Charset","UTF-8");
try(
//获取httpURLConnection的输入流
InputStream is = connection.getInputStream();
//输入流转成字符串,指定编码
InputStreamReader isr = new InputStreamReader(is, StandardCharsets.UTF_8);
BufferedReader br = new BufferedReader(isr);
){
String line;
while((line = br.readLine()) != null){
System.out.println(line);
}
}
}
httpclient发送get请求
引入依赖
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.13</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpmime</artifactId>
<version>4.5.13</version>
</dependency>
get无参请求
controller方法
@RequestMapping(value = "/test1",method = RequestMethod.GET)
public String test1(UserRequest userRequest, HttpServletRequest httpServletRequest){
System.out.println("业务入参:"+userRequest);
//获取所有请求头
Enumeration<String> headerNames = httpServletRequest.getHeaderNames();
while (headerNames.hasMoreElements()){
String headName = headerNames.nextElement();
System.out.println(headName+": "+httpServletRequest.getHeader(headName));
}
return "test1";
}
httpclient实现
@Test
public void testGet3() throws Exception {
//可关闭的httpClient客户端,相当于打开的浏览器
CloseableHttpClient httpClient = HttpClients.createDefault();
String urlStr = "http://localhost:8899/httpclient-demo/test1";
//构造get请求
HttpGet httpGet = new HttpGet(urlStr);
//响应对象
CloseableHttpResponse response = null;
try {
//执行get请求
response = httpClient.execute(httpGet);
} catch (IOException e) {
e.printStackTrace();
}finally {
if (httpClient != null){
try {
httpClient.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if(response != null){
try {
response.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
请求头的作用:防盗链
controller方法同上,httpclient访问方法:
@Test
public void testGet3() throws Exception {
//可关闭的httpClient客户端,相当于打开的浏览器
CloseableHttpClient httpClient = HttpClients.createDefault();
String urlStr = "https://www.baidu.com/";
//构造get请求
HttpGet httpGet = new HttpGet(urlStr);
//解决httpclient被认为不是真人行为
httpGet.addHeader("User-Agent:","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36");
//防盗链:value发生防盗链的网站的url
httpGet.addHeader("Referrer","https://www.baidu.com/");
//响应对象
CloseableHttpResponse response = null;
try {
//执行get请求
response = httpClient.execute(httpGet);
} catch (IOException e) {
e.printStackTrace();
}finally {
if (httpClient != null){
try {
httpClient.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if(response != null){
try {
response.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
get有参请求(需要urlencode编码)
controller方法同上,httpclient访问方法:
/**
* 使用httpClient发送get请求
* 3.有参的情况,需要urlencode编码
* 4.增加一些httClient的api认识
*/
@Test
public void testGet1() throws UnsupportedEncodingException {
//可关闭的httpClient客户端,相当于打开的浏览器
CloseableHttpClient httpClient = HttpClients.createDefault();
//构造httpGet请求对象
String passwordParam = "123+abb 456|789"; //接收参数:123 abb123 abb
//String passwordParam = "123 abb"; 报错 String passwordParam = "123|abb"; 报错
//表单提交get请求,浏览器会自动做url编码,web容器有自动解码的能力
//做urlencode:浏览器会自动做 123+abb 456|789 ---123%2Babb+456%7C789
passwordParam = URLEncoder.encode(passwordParam,StandardCharsets.UTF_8.name());
String urlStr = "http://localhost:8899/httpclient-demo/test1?userName=aaa&password="+passwordParam;
HttpGet httpGet = new HttpGet(urlStr);
//响应对象
CloseableHttpResponse response = null;
try {
//执行get请求
response = httpClient.execute(httpGet);
} catch (IOException e) {
e.printStackTrace();
}finally {
if (httpClient != null){
try {
httpClient.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if(response != null){
try {
response.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
获取响应头及相应的content-type
controller方法同上,httpclient访问方法:
@Test
public void testGet1() throws UnsupportedEncodingException {
//可关闭的httpClient客户端,相当于打开的浏览器
CloseableHttpClient httpClient = HttpClients.createDefault();
//构造httpGet请求对象
String passwordParam = "123+abb 456|789"; //接收参数:123 abb123 abb
//String passwordParam = "123 abb"; 报错 String passwordParam = "123|abb"; 报错
//表单提交get请求,浏览器会自动做url编码,web容器有自动解码的能力
//做urlencode:浏览器会自动做 123+abb 456|789 ---123%2Babb+456%7C789
passwordParam = URLEncoder.encode(passwordParam,StandardCharsets.UTF_8.name());
String urlStr = "http://localhost:8899/httpclient-demo/test1?userName=aaa&password="+passwordParam;
HttpGet httpGet = new HttpGet(urlStr);
//响应对象
CloseableHttpResponse response = null;
try {
//执行get请求
response = httpClient.execute(httpGet);
/**
获取响应头及相应的content-type
**/
//代表本次请求的成功,失败的状态
StatusLine statusLine = response.getStatusLine();
if(HttpStatus.SC_OK == statusLine.getStatusCode()){
System.out.println("响应成功!");
//获取响应头
Header[] headers = response.getAllHeaders();
for (Header header : headers) {
System.out.println("响应头:"+header.getName()+"的值:"+header.getValue());
}
/**获取响应结果:DecompressingEntity
* HttpEntity:不仅可以作为结果,也可以作为请求的参数实体,有很多的实现
*/
HttpEntity entity = response.getEntity();
System.out.println("ContentType:"+entity.getContentType());
//对HttpEntity操作的工具类
String result = EntityUtils.toString(entity, StandardCharsets.UTF_8);
System.out.println("result + "+result);
//确保流关闭,实体内容被消费
EntityUtils.consume(entity);
}else{
System.out.println("响应失败,响应码是:"+statusLine.getStatusCode());
}
} catch (IOException e) {
e.printStackTrace();
}finally {
if (httpClient != null){
try {
httpClient.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if(response != null){
try {
response.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
}
保存网络图片到本地
controller方法:
@PostMapping(value = "/test3")
public String test3(@RequestParam("fileName") MultipartFile[] multipartFiles, UserRequest userRequest, HttpServletRequest httpServletRequest){
for (MultipartFile multipartFile : multipartFiles) {
System.out.println("上传的文件名:"+multipartFile.getOriginalFilename());
}
System.out.println("业务入参:"+userRequest);
//获取所有请求头
Enumeration<String> headerNames = httpServletRequest.getHeaderNames();
while (headerNames.hasMoreElements()){
String headName = headerNames.nextElement();
System.out.println(headName+": "+httpServletRequest.getHeader(headName));
}
return "test3";
}
httpclient访问:
/**
* 获取网络上的图片并保存到本地
* @throws Exception
*/
@Test
public void testGet2() throws Exception {
CloseableHttpClient httpClient = HttpClients.createDefault();
//图片地址
String urlStr = "http://www.roadjava.com/thumbnail/2022/01/54560511366907187_seata.png";
HttpGet httpGet = new HttpGet(urlStr);
CloseableHttpResponse response = null;
try {
response = httpClient.execute(httpGet);
StatusLine statusLine = response.getStatusLine();
if(HttpStatus.SC_OK == statusLine.getStatusCode()){
HttpEntity entity = response.getEntity();
//image/jpg image/jpeg image/png image/jpg图片的后缀
String contentType = entity.getContentType().getValue();
String suffix= ".jpg";
if(contentType.contains("jpg")||contentType.contains("jpep")){
suffix = ".jpg";
}else if(contentType.contains("png")){
suffix = ".png";
}else if(contentType.contains("bmp")||contentType.contains("bitmap")){
suffix = ".bmp";
}else if(contentType.contains("gif")){
suffix = ".gif";
}
//获取文件的字节流
byte[] bytes = EntityUtils.toByteArray(entity);
//本地保存图片的路径
String filePath ="D:\\abc"+suffix;
//输出流写入到文件
FileOutputStream fos = new FileOutputStream(filePath);
fos.write(bytes);
fos.close();
EntityUtils.consume(entity);
}else{
System.out.println("响应失败,响应码是:"+statusLine.getStatusCode());
}
} catch (IOException e) {
e.printStackTrace();
}finally {
if (httpClient != null){
try {
httpClient.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if(response != null){
try {
response.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
设置访问代理
@Test
public void testGet3() throws Exception {
//可关闭的httpClient客户端,相当于打开的浏览器
CloseableHttpClient httpClient = HttpClients.createDefault();
String urlStr = "http://localhost:8899/httpclient-demo/test1?userName=aaa";
HttpGet httpGet = new HttpGet(urlStr);
//设置访问代理,避免爬虫被发现
//创建一个代理
String ip = "165.16.45.91";
int port = 7777;
HttpHost proxy = new HttpHost(ip,port);
//对每一个请求进行配置,会覆盖全局的默认请求配置
RequestConfig requestConfig = RequestConfig.custom().setProxy(proxy).build();
httpGet.setConfig(requestConfig);
System.out.println(":"+requestConfig);
//:[expectContinueEnabled=false, proxy=http://165.16.45.91:7777, localAddress=null, cookieSpec=null, redirectsEnabled=true,
// relativeRedirectsAllowed=true, maxRedirects=50, circularRedirectsAllowed=false, authenticationEnabled=true, targetPreferredAuthSchemes=null,
// proxyPreferredAuthSchemes=null, connectionRequestTimeout=-1, connectTimeout=-1, socketTimeout=-1, contentCompressionEnabled=true, normalizeUri=true]
//响应对象
CloseableHttpResponse response = null;
try {
//执行get请求
response = httpClient.execute(httpGet);
//代表本次请求的成功,失败的状态
StatusLine statusLine = response.getStatusLine();
if(HttpStatus.SC_OK == statusLine.getStatusCode()){
System.out.println("响应成功!");
/**获取响应结果:DecompressingEntity
* HttpEntity:不仅可以作为结果,也可以作为请求的参数实体,有很多的实现
*/
HttpEntity entity = response.getEntity();
//对HttpEntity操作的工具类
String result = EntityUtils.toString(entity, StandardCharsets.UTF_8);
System.out.println("result + "+result);
//确保流关闭,实体内容被消费
EntityUtils.consume(entity);
}else{
System.out.println("响应失败,响应码是:"+statusLine.getStatusCode());
}
} catch (IOException e) {
e.printStackTrace();
}finally {
if (httpClient != null){
try {
httpClient.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if(response != null){
try {
response.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
连接超时
controller方法:
@RequestMapping(value = "/test1",method = RequestMethod.GET)
public String test1(UserRequest userRequest, HttpServletRequest httpServletRequest){
System.out.println("业务入参:"+userRequest);
try {
//超时时间
Thread.sleep(5000);
} catch (InterruptedException e) {
e.printStackTrace();
}
//获取所有请求头
Enumeration<String> headerNames = httpServletRequest.getHeaderNames();
while (headerNames.hasMoreElements()){
String headName = headerNames.nextElement();
System.out.println(headName+": "+httpServletRequest.getHeader(headName));
}
return "test1";
}
httpclient访问:
/**
* 设置访问代理,避免爬虫被发现
* 设置连接超时和读取超时
* @throws Exception
*/
@Test
public void testGet3() throws Exception {
//可关闭的httpClient客户端,相当于打开的浏览器
CloseableHttpClient httpClient = HttpClients.createDefault();
String urlStr = "http://localhost:8899/httpclient-demo/test1?userName=aaa";
HttpGet httpGet = new HttpGet(urlStr);
//设置连接超时
RequestConfig requestConfig = RequestConfig.custom()
//设置连接超时 ms,完成tcp3次握手的时间上限
.setConnectTimeout(5000)
//读取超时,表示从请求的网址处获得响应数据的时间间隔
.setSocketTimeout(3000)
//指从连接池中获取connection连接的超时时间
//.setConnectionRequestTimeout(5000)
.build();
httpGet.setConfig(requestConfig);
//响应对象
CloseableHttpResponse response = null;
try {
//执行get请求
response = httpClient.execute(httpGet);
} catch (IOException e) {
e.printStackTrace();
}finally {
if (httpClient != null){
try {
httpClient.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if(response != null){
try {
response.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
发送post请求
MIME type和 Content Type
MIME Type:多用途互联网邮件扩展类型
信息头 | 含义 | 举例 |
---|---|---|
MIME-Version | MIME版本 | 1.0 |
Content-Type | 内容类型 | application/x-www-form-urlencoded,application/json |
Content-Transfer-Encoding | 编码格式 | 8bit,binary |
Content-Disposition | 内容排列方式 | 上传文件:Content-Disposition:form-data name=“fileName”;fileName=“d://file.xml” 下载文件:Content-Disposition:attachment;filename=URLEncoder(“xx.zip”,“UTF-8”) |
tomcat/conf/web.xml中可查看对应的mime-type的值
网页form表单enctype可用的MIME类型(Content-Type类型)
- application/x-www-form-urlencoded:表单类型
- multipart/form-data:传文件类型
- text/plain:字符串类型
表单类型的post请求
controller方法
@PostMapping(value = "/test2")
public String test2(UserRequest userRequest, HttpServletRequest httpServletRequest){
System.out.println("其他业务入参:"+userRequest);
//获取所有请求头
Enumeration<String> headerNames = httpServletRequest.getHeaderNames();
while (headerNames.hasMoreElements()){
String headName = headerNames.nextElement();
System.out.println(headName+": "+httpServletRequest.getHeader(headName));
}
return "test2";
}
httpclient访问:
/**
* 发送application/x-www-form-urlencoded类型的post请求,默认模式
*/
@Test
public void testPost1() throws Exception {
//可关闭的httpClient客户端,相当于打开的浏览器
CloseableHttpClient httpClient = HttpClients.createDefault();
//构造httpPost请求对象
String urlStr = "http://localhost:8899/httpclient-demo/test2";
HttpPost httpPost = new HttpPost(urlStr);
//设置请求头
httpPost.setHeader("Content-Type","application/x-www-form-urlencoded; charset=UTF-8");
/**
* NameValuePair:name和Input标签里输入的值,构成了一个nameValuePair对象
*/
ArrayList<NameValuePair> list = new ArrayList<>();
list.add(new BasicNameValuePair("userName","aaa"));
list.add(new BasicNameValuePair("password","123456"));
//把参数集合设置到formEntity
UrlEncodedFormEntity formEntity = new UrlEncodedFormEntity(list, Consts.UTF_8);
httpPost.setEntity(formEntity);
//响应对象
CloseableHttpResponse response = null;
try {
//执行get请求
response = httpClient.execute(httpPost);
//代表本次请求的成功,失败的状态
StatusLine statusLine = response.getStatusLine();
if(HttpStatus.SC_OK == statusLine.getStatusCode()){
System.out.println("响应成功!");
//获取响应头
Header[] headers = response.getAllHeaders();
for (Header header : headers) {
System.out.println("响应头:"+header.getName()+"的值:"+header.getValue());
}
/**获取响应结果:DecompressingEntity
* HttpEntity:不仅可以作为结果,也可以作为请求的参数实体,有很多的实现
*/
HttpEntity entity = response.getEntity();
System.out.println("ContentType:"+entity.getContentType());
//对HttpEntity操作的工具类
String result = EntityUtils.toString(entity, StandardCharsets.UTF_8);
System.out.println("result + "+result);
//确保流关闭,实体内容被消费
EntityUtils.consume(entity);
}else{
System.out.println("响应失败,响应码是:"+statusLine.getStatusCode());
}
} catch (IOException e) {
e.printStackTrace();
}finally {
if (httpClient != null){
try {
httpClient.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if(response != null){
try {
response.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
json类型的post请求
controller方法
@PostMapping(value = "/testJson")
public String testJson(@RequestBody UserRequest userRequest, HttpServletRequest httpServletRequest){
System.out.println("业务入参:"+userRequest);
//获取所有请求头
Enumeration<String> headerNames = httpServletRequest.getHeaderNames();
while (headerNames.hasMoreElements()){
String headName = headerNames.nextElement();
System.out.println(headName+": "+httpServletRequest.getHeader(headName));
}
return "testJson";
}
httpclient访问:
/**
* 发送application/json post请求
* @throws Exception
*/
@Test
public void testPost2() throws Exception {
//可关闭的httpClient客户端,相当于打开的浏览器
CloseableHttpClient httpClient = HttpClients.createDefault();
//构造httpPost请求对象
String urlStr = "http://localhost:8899/httpclient-demo/testJson";
HttpPost httpPost = new HttpPost(urlStr);
//设置传入数据
JSONObject obj = new JSONObject();
obj.put("userName","admin");
obj.put("password","123456");
StringEntity stringEntity = new StringEntity(obj.toString(),Consts.UTF_8);
//给entity设置内容类型
stringEntity.setContentType(new BasicHeader("Content-Type","application/json; charset=UTF-8"));
//设置entity编码,避免中文乱码
stringEntity.setContentEncoding(Consts.UTF_8.name());
httpPost.setEntity(stringEntity);
//响应对象
CloseableHttpResponse response = null;
try {
//执行get请求
response = httpClient.execute(httpPost);
//代表本次请求的成功,失败的状态
StatusLine statusLine = response.getStatusLine();
if(HttpStatus.SC_OK == statusLine.getStatusCode()){
System.out.println("响应成功!");
//获取响应头
Header[] headers = response.getAllHeaders();
for (Header header : headers) {
System.out.println("响应头:"+header.getName()+"的值:"+header.getValue());
}
/**获取响应结果:DecompressingEntity
* HttpEntity:不仅可以作为结果,也可以作为请求的参数实体,有很多的实现
*/
HttpEntity entity = response.getEntity();
System.out.println("ContentType:"+entity.getContentType());
//对HttpEntity操作的工具类
String result = EntityUtils.toString(entity, StandardCharsets.UTF_8);
System.out.println("result + "+result);
//确保流关闭,实体内容被消费
EntityUtils.consume(entity);
}else{
System.out.println("响应失败,响应码是:"+statusLine.getStatusCode());
}
} catch (IOException e) {
e.printStackTrace();
}finally {
if (httpClient != null){
try {
httpClient.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if(response != null){
try {
response.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
上传文件的post请求
controller方法:
@PostMapping(value = "/test3")
public String test3(@RequestParam("fileName") MultipartFile[] multipartFiles, UserRequest userRequest, HttpServletRequest httpServletRequest){
for (MultipartFile multipartFile : multipartFiles) {
System.out.println("上传的文件名:"+multipartFile.getOriginalFilename());
}
System.out.println("业务入参:"+userRequest);
//获取所有请求头
Enumeration<String> headerNames = httpServletRequest.getHeaderNames();
while (headerNames.hasMoreElements()){
String headName = headerNames.nextElement();
System.out.println(headName+": "+httpServletRequest.getHeader(headName));
}
return "test3";
}
httpclient访问:
@Test
public void testPost3() throws Exception {
//可关闭的httpClient客户端,相当于打开的浏览器
CloseableHttpClient httpClient = HttpClients.createDefault();
//构造httpPost请求对象
String urlStr = "http://localhost:8899/httpclient-demo/test3";
HttpPost httpPost = new HttpPost(urlStr);
//构造ContentBody的实现类对象
FileBody fileBody = new FileBody(new File("C:\\Users\\86155\\Desktop\\项目个人周报.xlsx"));
//构造上传文件使用的entity
MultipartEntityBuilder builder = MultipartEntityBuilder.create();
builder.setCharset(Consts.UTF_8);//设置编码
//builder.setContentType(ContentType.MULTIPART_FORM_DATA);//设置内容类型
builder.setContentType(ContentType.create("multipart/form-data",Consts.UTF_8));
builder.setMode(HttpMultipartMode.BROWSER_COMPATIBLE);//设置浏览器模式
//添加参数,对于普通的表单字段如果含有中文的话,不能通过addTextBody,否则乱码
StringBody userNameText = new StringBody("小米",ContentType.create("text/plain",Consts.UTF_8));
HttpEntity postEntity = builder.addPart("fileName", fileBody)
//通过file,byte[],inputstream来上传文件
.addBinaryBody("fileName", new File("C:\\Users\\86155\\Desktop\\fm.jpg"))
//.addTextBody("userName", "张三")
.addPart("userName",userNameText)
.addTextBody("password", "1223456")
.build();
httpPost.setEntity(postEntity);
//响应对象
CloseableHttpResponse response = null;
try {
//执行get请求
response = httpClient.execute(httpPost);
//代表本次请求的成功,失败的状态
StatusLine statusLine = response.getStatusLine();
if(HttpStatus.SC_OK == statusLine.getStatusCode()){
System.out.println("响应成功!");
//获取响应头
Header[] headers = response.getAllHeaders();
for (Header header : headers) {
System.out.println("响应头:"+header.getName()+"的值:"+header.getValue());
}
/**获取响应结果:DecompressingEntity
* HttpEntity:不仅可以作为结果,也可以作为请求的参数实体,有很多的实现
*/
HttpEntity entity = response.getEntity();
System.out.println("ContentType:"+entity.getContentType());
//对HttpEntity操作的工具类
String result = EntityUtils.toString(entity, StandardCharsets.UTF_8);
System.out.println("result + "+result);
//确保流关闭,实体内容被消费
EntityUtils.consume(entity);
}else{
System.out.println("响应失败,响应码是:"+statusLine.getStatusCode());
}
} catch (IOException e) {
e.printStackTrace();
}finally {
if (httpClient != null){
try {
httpClient.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if(response != null){
try {
response.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}