文件上传下载系列
HTTP Component 5.0 + 教程
介绍
超文本传输协议 (HTTP) 可能是当今 Internet 上使用的最重要的协议。 Web 服务、支持网络的设备和网络计算的增长继续将 HTTP 协议的作用扩展到用户驱动的 Web 浏览器之外,同时增加了需要 HTTP 支持的应用程序的数量。 尽管 java.net 包提供了通过 HTTP 访问资源的基本功能,但它并没有提供许多应用程序所需的全部灵活性或功能。 HttpClient 旨在通过提供高效、最新且功能丰富的包来实现最新 HTTP 标准和建议的客户端来填补这一空白。 为扩展而设计,同时为基本 HTTP 协议提供强大的支持,任何构建 HTTP 感知客户端应用程序(如 Web 浏览器、Web 服务客户端或利用或扩展 HTTP 协议进行分布式通信的系统)的任何人都可能对 HttpClient 感兴趣。
MAVEN 引用
<dependency>
<groupId>org.apache.httpcomponents.client5</groupId>
<artifactId>httpclient5</artifactId>
<version>5.1.3</version>
</dependency>
使用示例
同步
package com.allens.http.pool;
/**
* HttpClientUtil
*
* @author 澄风
* @date 2021/6/21
*/
@SuppressWarnings("all")
public class HttpClientUtil {
private static CloseableHttpClient httpClient = null;
static {
PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager();
// 总连接池连接数量
connectionManager.setMaxTotal(1500);
// socket config 可自定义
// httpclient 现在的默认socket超时时间是3分钟,如果需要调整请自己定义socket config进行定义
// private static final Timeout DEFAULT_SOCKET_TIMEOUT = Timeout.ofMinutes(3);
connectionManager.setDefaultSocketConfig(SocketConfig.DEFAULT);
// 可为每个域名设置最大的并行连接数量
// connectionManager.setMaxPerRoute(new HttpRoute(new HttpHost("xx.xx.xx.xx")), 80);
connectionManager.setDefaultMaxPerRoute(100);
// setConnectTimeout:设置建立连接的超时时间
// setConnectionRequestTimeout:从连接池中拿连接的等待超时时间
// setSocketTimeout:发出请求后等待对端应答的超时时间
RequestConfig requestConfig = RequestConfig.custom()
// 保持长连接
.setConnectionKeepAlive(TimeValue.of(600, TimeUnit.SECONDS))
// 连接超时时间
.setConnectTimeout(Timeout.of(1000, TimeUnit.SECONDS))
// 请求超时时间
.setConnectionRequestTimeout(Timeout.of(1000, TimeUnit.SECONDS))
// 和上面二选一
// .setDefaultKeepAlive(1000, TimeUnit.SECONDS)
.build();
// 重试处理器,StandardHttpRequestRetryHandler
// HttpRequestRetryHandler retryHandler = new StandardHttpRequestRetryHandler();
// BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider();
// credentialsProvider.setCredentials(new AuthScope());
httpClient = HttpClients.custom()
.setConnectionManager(connectionManager)
.setDefaultRequestConfig(requestConfig)
// .setDefaultCredentialsProvider(credentialsProvider)
// .setRetryHandler(retryHandler)
.build();
}
/**
* GET请求
*
* @param uri
* @param getParams
* @return
*/
public static String doHttpGet(String uri, Map<String, String> getParams) {
TestUtils testUtils = new TestUtils();
String s = testUtils.testSome();
System.out.println(s);
CloseableHttpResponse response = null;
try {
URIBuilder uriBuilder = new URIBuilder(uri);
if (null != getParams && !getParams.isEmpty()) {
List<NameValuePair> list = new ArrayList<>();
for (Map.Entry<String, String> param : getParams.entrySet()) {
list.add(new BasicNameValuePair(param.getKey(), param.getValue()));
}
uriBuilder.setParameters(list);
}
HttpGet httpGet = new HttpGet(uriBuilder.build());
response = httpClient.execute(httpGet);
int statusCode = response.getCode();
if (HttpStatus.SC_OK == statusCode) {
HttpEntity entity = response.getEntity();
if (null != entity) {
String resStr = EntityUtils.toString(entity, "utf-8");
return resStr;
}
}
} catch (Exception e) {
e.printStackTrace();
// log.error("CloseableHttpClient-get-请求异常", e);
} finally {
try {
if (null != response)
response.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return "FAIL:NO CONTENT";
}
/**
* GET请求
*
* @param uri
* @param getParams
* @return
*/
public static JSONObject doHttpDelete(String uri, Map<String, String> getParams) {
TestUtils testUtils = new TestUtils();
String s = testUtils.testSome();
System.out.println(s);
CloseableHttpResponse response = null;
try {
URIBuilder uriBuilder = new URIBuilder(uri);
if (null != getParams && !getParams.isEmpty()) {
List<NameValuePair> list = new ArrayList<>();
for (Map.Entry<String, String> param : getParams.entrySet()) {
list.add(new BasicNameValuePair(param.getKey(), param.getValue()));
}
uriBuilder.setParameters(list);
}
HttpDelete httpDelete = new HttpDelete(uriBuilder.build());
response = httpClient.execute(httpDelete);
int statusCode = response.getCode();
if (HttpStatus.SC_OK == statusCode) {
HttpEntity entity = response.getEntity();
if (null != entity) {
String resStr = EntityUtils.toString(entity, "utf-8");
return JSON.parseObject(resStr);
}
}
} catch (Exception e) {
e.printStackTrace();
// log.error("CloseableHttpClient-get-请求异常", e);
} finally {
try {
if (null != response)
response.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return new JSONObject();
}
/**
* 下载文件
*
* @param uri
* @param getParams
* @return
*/
public static void doHttpGetForFile(String uri,
Map<String, String> getParams,
Map<String, String> headers,
long offset,
File destFile) {
CloseableHttpResponse response = null;
long countLong = 0;
try {
URIBuilder uriBuilder = new URIBuilder(uri);
if (null != getParams && !getParams.isEmpty()) {
List<NameValuePair> list = new ArrayList<>();
for (Map.Entry<String, String> param : getParams.entrySet()) {
list.add(new BasicNameValuePair(param.getKey(), param.getValue()));
}
uriBuilder.setParameters(list);
}
HttpGet httpGet = new HttpGet(uriBuilder.build());
Optional.ofNullable(headers).ifPresent(data -> {
data.forEach((k,v) -> {
httpGet.addHeader(k, v);
});
});
response = httpClient.execute(httpGet);
int statusCode = response.getCode();
Header header = response.getHeader("Content-Length");
if (HttpStatus.SC_OK == statusCode || HttpStatus.SC_PARTIAL_CONTENT == statusCode) {
HttpEntity entity = response.getEntity();
if (null != entity) {
try (InputStream content = entity.getContent();
FileOutputStream fileOutputStream = new FileOutputStream(destFile, true);
OutputStream out = new BufferedOutputStream(fileOutputStream)) {
byte[] buffer = new byte[102400]; // 100K
int length = 0;
while ((length = content.read(buffer, 0, buffer.length)) != -1) {
countLong += length;
out.write(buffer, 0, length);
//Thread.sleep(100);
//break;
}
System.out.println("Count long :" + countLong);
} catch (Exception e) {
e.printStackTrace();
}
}
} else if (HttpStatus.SC_ACCEPTED == statusCode) {
System.out.println("已经下载过了,请勿重新下载...");
}
} catch (Exception e) {
e.printStackTrace();
// log.error("CloseableHttpClient-get-请求异常", e);
} finally {
try {
if (null != response)
response.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
/**
* post表单提交
*
* @param uri
* @param getParams
* @return
*/
public static JSONObject doHttpPostForForm(String uri, Map<String, String> getParams) {
CloseableHttpResponse response = null;
try {
HttpPost httpPost = new HttpPost(uri);
if (null != getParams && !getParams.isEmpty()) {
List<NameValuePair> list = new ArrayList<>();
for (Map.Entry<String, String> param : getParams.entrySet()) {
list.add(new BasicNameValuePair(param.getKey(), param.getValue()));
}
HttpEntity httpEntity = new UrlEncodedFormEntity(list, StandardCharsets.UTF_8);
httpPost.setEntity(httpEntity);
}
response = httpClient.execute(httpPost);
int statusCode = response.getCode();
if (HttpStatus.SC_OK == statusCode) {
HttpEntity entity = response.getEntity();
if (null != entity) {
String resStr = EntityUtils.toString(entity, "utf-8");
return JSON.parseObject(resStr);
}
}
} catch (Exception e) {
// log.error("CloseableHttpClient-post-请求异常", e);
} finally {
try {
if (null != response)
response.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return new JSONObject();
}
/**
* post body请求
*
* @param uri
* @param getParams
* @return
*/
public static JSONObject doHttpPost(String uri, Map<String, String> getParams, Map<String, String> headers) {
CloseableHttpResponse response = null;
try {
HttpPost httpPost = new HttpPost(uri);
String queryString = JSON.toJSONString(getParams, SerializerFeature.valueOf("utf-8"));
StringEntity stringEntity = new StringEntity(queryString);
httpPost.setEntity(stringEntity);
// 设置header
Optional.ofNullable(headers).ifPresent(data -> data.forEach(httpPost::setHeader));
response = httpClient.execute(httpPost);
int statusCode = response.getCode();
if (HttpStatus.SC_OK == statusCode) {
HttpEntity entity = response.getEntity();
if (null != entity) {
String resStr = EntityUtils.toString(entity, "utf-8");
return JSON.parseObject(resStr);
}
}
} catch (Exception e) {
// log.error("CloseableHttpClient-post-请求异常", e);
} finally {
try {
if (null != response)
response.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return new JSONObject();
}
/**
* 上传文件表单
*
* @param uri
* @param getParams
* @return
*/
public static String doHttpPostForFormMutipart(String uri, Map<String, String> getParams, File file) {
CloseableHttpResponse response = null;
try {
HttpPost httpPost = new HttpPost(uri);
// httpPost.setHeader(new BasicHeader("Content-Type", "multipart/form-data"));
if (null != getParams && !getParams.isEmpty()) {
List<NameValuePair> list = new ArrayList<>();
for (Map.Entry<String, String> param : getParams.entrySet()) {
list.add(new BasicNameValuePair(param.getKey(), param.getValue()));
}
}
MultipartPart multipartPart = MultipartPartBuilder
.create()
.setBody(new FileBody(file, ContentType.IMAGE_JPEG))
.addHeader("content-type", "image/jpeg")
.build();
org.apache.hc.core5.http.HttpEntity httpEntity = MultipartEntityBuilder.create()
//.addPart(multipartPart)
.addPart("file", new FileBody(file, ContentType.IMAGE_JPEG))
//.setContentType(ContentType.IMAGE_JPEG)
.build();
// HttpEntity httpEntity = new UrlEncodedFormEntity(list, "utf-8");
httpPost.setEntity(httpEntity);
response = httpClient.execute(httpPost);
int statusCode = response.getCode();
if (HttpStatus.SC_OK == statusCode) {
HttpEntity entity = response.getEntity();
if (null != entity) {
String resStr = EntityUtils.toString(entity, "utf-8");
return resStr;
}
} else {
HttpEntity entity = response.getEntity();
System.out.println(EntityUtils.toString(entity, "utf-8"));
}
} catch (Exception e) {
e.printStackTrace();
//log.error("CloseableHttpClient-post-请求异常", e);
} finally {
try {
if (null != response)
response.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return new String("NO CONTENT");
}
}
- GET请求
普通GET请求http://xxx.xx?queryParam=xxxxx&a=b - 下载文件
下载文件,支持断点续传,但需要服务器也支持,有兴趣的话可以看我的另一篇文章断点续传。
/**
* 下载文件
*/
@Test
public void testDownloadFile () {
Map<String, String> headers = Maps.newHashMap();
long currentOffset = 0;
File destFile = new File("/Users/yueyu/Project/allens-learn/download/123456.jpg");
if (destFile.exists() && destFile.length() > 0) {
currentOffset = destFile.length();
}
// 断点下载的核心
headers.put("range", "bytes=" + currentOffset + "-");
HttpClientUtil.doHttpGetForFile(
// "http://localhost:8091/download/media?id=1631503405304",
"http://localhost:8091/download/update",
null,
headers,
currentOffset,
destFile);
}
- post表单提交
普通的表单提交content-type=multipart/form-data
- post body请求
普通的http body提交content-type=application/json
- 上传文件表单
异步
/**
* Example of asynchronous HTTP/1.1 request execution.
*/
public class AsyncClientHttpExchange {
public static void main(final String[] args) throws Exception {
final IOReactorConfig ioReactorConfig = IOReactorConfig.custom()
.setSoTimeout(Timeout.ofSeconds(5))
.build();
final CloseableHttpAsyncClient client = HttpAsyncClients.custom()
.setIOReactorConfig(ioReactorConfig)
.build();
client.start();
final HttpHost target = new HttpHost("httpbin.org");
final String[] requestUris = new String[] {"/", "/ip", "/user-agent", "/headers"};
for (final String requestUri: requestUris) {
final SimpleHttpRequest request = SimpleRequestBuilder.get()
.setHttpHost(target)
.setPath(requestUri)
.build();
System.out.println("Executing request " + request);
final Future<SimpleHttpResponse> future = client.execute(
SimpleRequestProducer.create(request),
SimpleResponseConsumer.create(),
new FutureCallback<SimpleHttpResponse>() {
@Override
public void completed(final SimpleHttpResponse response) {
System.out.println(request + "->" + new StatusLine(response));
System.out.println(response.getBody());
}
@Override
public void failed(final Exception ex) {
System.out.println(request + "->" + ex);
}
@Override
public void cancelled() {
System.out.println(request + " cancelled");
}
});
future.get();
}
System.out.println("Shutting down");
client.close(CloseMode.GRACEFUL);
}
}
代理
public void proxy () {
//创建httpClient实例
CloseableHttpClient httpClient = HttpClients.createDefault();
/**
* httpClient.getHostConfiguration().setProxy("192.168.101.1", 5608);
* httpClient.getParams().setAuthenticationPreemptive(true);
* //如果代理需要密码验证,这里设置用户名密码
* httpClient.getState().setProxyCredentials(AuthScope.ANY, new UsernamePasswordCredentials("llying.iteye.com","llying"));
*/
//创建httpGet实例
HttpGet httpGet = new HttpGet("http://www.tuicool.com");
//设置代理IP,设置连接超时时间 、 设置 请求读取数据的超时时间 、 设置从connect Manager获取Connection超时时间、
HttpHost proxy = new HttpHost("58.60.255.82",8118);
RequestConfig requestConfig = RequestConfig.custom()
.setProxy(proxy)
.setConnectTimeout(Timeout.ofMilliseconds(1000))
// .setSocketTimeout(10000)
.setConnectionRequestTimeout(Timeout.ofMilliseconds(3000))
.build();
httpGet.setConfig(requestConfig);
//设置请求头消息
httpGet.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36");
try {
CloseableHttpResponse response = null;
response = httpClient.execute(httpGet);
if (response != null) {
HttpEntity entity = response.getEntity(); //获取返回实体
if (entity != null) {
System.out.println("网页内容为:"+ EntityUtils.toString(entity,"utf-8"));
}
}
if (response != null){
response.close();
}
if (httpClient != null){
httpClient.close();
}
} catch (IOException | ParseException e) {
e.printStackTrace();
}
}
maxDefaultPerRoute
每个路由默认的最大连接数。这里路由的定义是ip+port。ip和port其中一个不一样就是不同的路有。这意味某一个时刻,相同路由的连接数不能超过这个值(这里是每个路由的default值,实际可根据某个路由特殊设置具体的个性化值)。相当于在连接池中又划分一个个路由池。maxTotal
: 连接池的总共最大连接数。新的请求进来时,连接池中已有连接不可用时,那就只能新建一个连接,但是如果连接池中连接数已经达到上限,那么无法新建连接,意味着这个请求只能等待。
编程层面上经常会用到几种超时的设置。
1.connection timeout
: 属于网络协议上的概念,指的是3次握手建立连接的超时。
2.socket timeout
: 属于网络协议上的概念,指的是3次握手建立连接,client 发出请求开始到收到响应的这段时间的超时,其实是read timeout。
3.connection request timeout
:在apache http client,这个其实非网络协议上概念,属于自定义,表示的是从连接池获取连接的超时。
参考文章
httpclient的两个重要的参数maxPerRoute及MaxTotal
HttpClient 代理使用 proxy & 异步HttpClient大量请求