1. 为什么要整合HttpClient
RestTemplate是Spring自带的一个调用rest服务的客户端,它提供了多种便捷访问远程Http服务的方法,能够大大提高客户端的编写效率。
RestTemplate默认是使用JDK原生的URLConnection,默认超时为-1, 也就表示是没有超时时间的,这个肯定不能满足复杂情况的使用需求, restTemplate的工厂是支持使用HttpClient和OkHttp来作为客户端实现的
2. 为什么要使用连接池
在调用rest请求时,每次请求都需要和服务端建立连接,也就是三次握手,这是一个费时费力的工作,如果我们需要频繁对一个服务端进行调用,难道需要一直去建立连接吗?
所以使用连接池,可以避免多次建立连接的操作,节省资源开支
3. 依赖
除了spring的基本依赖以外,还需要准备下面几个依赖:
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.7</version>
</dependency>
<!--为了使用@ConfigurationProperties,还需要这个依赖-->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-configuration-processor</artifactId>
<optional>true</optional>
<version>${springboot.version}</version>
</dependency>
4. 配置类
package com.zgd.springboot.demo.template.config;
import com.alibaba.fastjson.JSON;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.Header;
import org.apache.http.HeaderElement;
import org.apache.http.HeaderElementIterator;
import org.apache.http.HttpHost;
import org.apache.http.client.HttpClient;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.conn.ConnectionKeepAliveStrategy;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.message.BasicHeader;
import org.apache.http.message.BasicHeaderElementIterator;
import org.apache.http.protocol.HTTP;
import org.apache.http.ssl.SSLContextBuilder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.http.client.ClientHttpRequestFactory;
import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;
import org.springframework.http.converter.HttpMessageConverter;
import org.springframework.http.converter.StringHttpMessageConverter;
import org.springframework.stereotype.Component;
import org.springframework.web.client.DefaultResponseErrorHandler;
import org.springframework.web.client.RestTemplate;
import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.SSLContext;
import java.nio.charset.Charset;
import java.security.KeyManagementException;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
/**
* @Author: zgd
* @Date: 2019/3/25 16:53
* @Description:
*/
@Configuration
@Component
@ConditionalOnClass(value = {RestTemplate.class, CloseableHttpClient.class})
@Data
@Slf4j
public class HttpClientConfig {
@Autowired
private HttpClientPoolConfig httpClientPoolConfig;
@Bean
public RestTemplate restTemplate() {
return new RestTemplate();
}
/**
* 创建HTTP客户端工厂
*/
@Bean(name = "clientHttpRequestFactory")
public ClientHttpRequestFactory clientHttpRequestFactory() {
/**
* maxTotalConnection 和 maxConnectionPerRoute 必须要配
*/
if (httpClientPoolConfig.getMaxTotalConnect() <= 0) {
throw new IllegalArgumentException("invalid maxTotalConnection: " + httpClientPoolConfig.getMaxTotalConnect());
}
if (httpClientPoolConfig.getMaxConnectPerRoute() <= 0) {
throw new IllegalArgumentException("invalid maxConnectionPerRoute: " + httpClientPoolConfig.getMaxConnectPerRoute());
}
HttpComponentsClientHttpRequestFactory clientHttpRequestFactory = new HttpComponentsClientHttpRequestFactory(httpClient());
// 连接超时
clientHttpRequestFactory.setConnectTimeout(httpClientPoolConfig.getConnectTimeout());
// 数据读取超时时间,即SocketTimeout
clientHttpRequestFactory.setReadTimeout(httpClientPoolConfig.getReadTimeout());
// 从连接池获取请求连接的超时时间,不宜过长,必须设置,比如连接不够用时,时间过长将是灾难性的
clientHttpRequestFactory.setConnectionRequestTimeout(httpClientPoolConfig.getConnectionRequestTimout());
return clientHttpRequestFactory;
}
/**
* 初始化RestTemplate,并加入spring的Bean工厂,由spring统一管理
*/
@Bean(name = "httpClientTemplate")
public RestTemplate restTemplate(ClientHttpRequestFactory factory) {
return createRestTemplate(factory);
}
/**
* 初始化支持异步的RestTemplate,并加入spring的Bean工厂,由spring统一管理,如果你用不到异步,则无须创建该对象
* 这个类过时了
* @return
*/
/* @Bean(name = "asyncRestTemplate")
@ConditionalOnMissingBean(AsyncRestTemplate.class)
public AsyncRestTemplate asyncRestTemplate(RestTemplate restTemplate) {
final Netty4ClientHttpRequestFactory factory = new Netty4ClientHttpRequestFactory();
factory.setConnectTimeout(this.connectionTimeout);
factory.setReadTimeout(this.readTimeout);
return new AsyncRestTemplate(factory, restTemplate);
}*/
/**
* 配置httpClient
*
* @return
*/
@Bean
public HttpClient httpClient() {
HttpClientBuilder httpClientBuilder = HttpClientBuilder.create();
try {
//设置信任ssl访问
SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, (arg0, arg1) -> true).build();
httpClientBuilder.setSSLContext(sslContext);
HostnameVerifier hostnameVerifier = NoopHostnameVerifier.INSTANCE;
SSLConnectionSocketFactory sslConnectionSocketFactory = new SSLConnectionSocketFactory(sslContext, hostnameVerifier);
Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
// 注册http和https请求
.register("http", PlainConnectionSocketFactory.getSocketFactory())
.register("https", sslConnectionSocketFactory).build();
//使用Httpclient连接池的方式配置(推荐),同时支持netty,okHttp以及其他http框架
PoolingHttpClientConnectionManager poolingHttpClientConnectionManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
// 最大连接数
poolingHttpClientConnectionManager.setMaxTotal(httpClientPoolConfig.getMaxTotalConnect());
// 同路由并发数
poolingHttpClientConnectionManager.setDefaultMaxPerRoute(httpClientPoolConfig.getMaxConnectPerRoute());
//配置连接池
httpClientBuilder.setConnectionManager(poolingHttpClientConnectionManager);
// 重试次数
httpClientBuilder.setRetryHandler(new DefaultHttpRequestRetryHandler(httpClientPoolConfig.getRetryTimes(), true));
//设置默认请求头
List<Header> headers = getDefaultHeaders();
httpClientBuilder.setDefaultHeaders(headers);
//设置长连接保持策略
httpClientBuilder.setKeepAliveStrategy(connectionKeepAliveStrategy());
return httpClientBuilder.build();
} catch (KeyManagementException | NoSuchAlgorithmException | KeyStoreException e) {
log.error("初始化HTTP连接池出错", e);
}
return null;
}
/**
* 配置长连接保持策略
* @return
*/
public ConnectionKeepAliveStrategy connectionKeepAliveStrategy(){
return (response, context) -> {
// Honor 'keep-alive' header
HeaderElementIterator it = new BasicHeaderElementIterator(
response.headerIterator(HTTP.CONN_KEEP_ALIVE));
while (it.hasNext()) {
HeaderElement he = it.nextElement();
log.info("HeaderElement:{}", JSON.toJSONString(he));
String param = he.getName();
String value = he.getValue();
if (value != null && "timeout".equalsIgnoreCase(param)) {
try {
return Long.parseLong(value) * 1000;
} catch(NumberFormatException ignore) {
log.error("解析长连接过期时间异常",ignore);
}
}
}
HttpHost target = (HttpHost) context.getAttribute(
HttpClientContext.HTTP_TARGET_HOST);
//如果请求目标地址,单独配置了长连接保持时间,使用该配置
Optional<Map.Entry<String, Integer>> any = Optional.ofNullable(httpClientPoolProperties.getKeepAliveTargetHost()).orElseGet(HashMap::new)
.entrySet().stream().filter(
e -> e.getKey().equalsIgnoreCase(target.getHostName())).findAny();
//否则使用默认长连接保持时间
return any.map(en -> en.getValue() * 1000L).orElse(httpClientPoolProperties.getKeepAliveTime() * 1000L);
};
}
/**
* 设置请求头
*
* @return
*/
private List<Header> getDefaultHeaders() {
List<Header> headers = new ArrayList<>();
headers.add(new BasicHeader("User-Agent",
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.16 Safari/537.36"));
headers.add(new BasicHeader("Accept-Encoding", "gzip,deflate"));
headers.add(new BasicHeader("Accept-Language", "zh-CN"));
headers.add(new BasicHeader("Connection", "Keep-Alive"));
return headers;
}
private RestTemplate createRestTemplate(ClientHttpRequestFactory factory) {
RestTemplate restTemplate = new RestTemplate(factory);
//我们采用RestTemplate内部的MessageConverter
//重新设置StringHttpMessageConverter字符集,解决中文乱码问题
modifyDefaultCharset(restTemplate);
//设置错误处理器
restTemplate.setErrorHandler(new DefaultResponseErrorHandler());
return restTemplate;
}
/**
* 修改默认的字符集类型为utf-8
*
* @param restTemplate
*/
private void modifyDefaultCharset(RestTemplate restTemplate) {
List<HttpMessageConverter<?>> converterList = restTemplate.getMessageConverters();
HttpMessageConverter<?> converterTarget = null;
for (HttpMessageConverter<?> item : converterList) {
if (StringHttpMessageConverter.class == item.getClass()) {
converterTarget = item;
break;
}
}
if (null != converterTarget) {
converterList.remove(converterTarget);
}
Charset defaultCharset = Charset.forName(httpClientPoolConfig.getCharset());
converterList.add(1, new StringHttpMessageConverter(defaultCharset));
}
}
这里有个注意点,就是RestTemplate的bean命名,建议和restTemplate区分开, 这样我们如果需要原生的restTemplate,就用restTemplate
,如果用我们自定义的,就用httpClientTemplate
@Resource
private RestTemplate restTemplate;
@Resource
private RestTemplate httpClientTemplate;
连接池的配置类:
这里如果没有上面的spring-boot-configuration-processor
这个依赖,会报错
package com.zgd.springboot.demo.template.config;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
import java.util.Map;
/**
* @Author: zgd
* @Date: 2019/3/26 19:00
* @Description:
*/
@Component
@ConfigurationProperties(prefix = "spring.http-client.pool")
@Data
public class HttpClientPoolConfig {
/**
* java配置的优先级低于yml配置;如果yml配置不存在,会采用java配置
*/
/**
* 连接池的最大连接数
*/
private int maxTotalConnect ;
/**
* 同路由的并发数
*/
private int maxConnectPerRoute ;
/**
* 客户端和服务器建立连接超时,默认2s
*/
private int connectTimeout = 2 * 1000;
/**
* 指客户端从服务器读取数据包的间隔超时时间,不是总读取时间,默认30s
*/
private int readTimeout = 30 * 1000;
private String charset = "UTF-8";
/**
* 重试次数,默认2次
*/
private int retryTimes = 2;
/**
* 从连接池获取连接的超时时间,不宜过长,单位ms
*/
private int connectionRequestTimout = 200;
/**
* 针对不同的地址,特别设置不同的长连接保持时间
*/
private Map<String,Integer> keepAliveTargetHost;
/**
* 针对不同的地址,特别设置不同的长连接保持时间,单位 s
*/
private int keepAliveTime = 60;
}
application.yml:
spring:
# yml配置的优先级高于java配置;如果yml配置和java配置同时存在,则yml配置会覆盖java配置
http-client:
pool:
#连接池的最大连接数,0代表不限;如果取0,需要考虑连接泄露导致系统崩溃的后果
maxTotalConnect: 1000
#每个路由的最大连接数,如果只调用一个地址,可以将其设置为最大连接数
maxConnectPerRoute: 200
# 指客户端和服务器建立连接的超时时间,ms , 最大约21秒,因为内部tcp在进行三次握手建立连接时,默认tcp超时时间是20秒
connectTimeout: 3000
# 指客户端从服务器读取数据包的间隔超时时间,不是总读取时间,也就是socket timeout,ms
readTimeout: 5000
# 从连接池获取连接的timeout,不宜过大,ms
connectionRequestTimout: 200
# 重试次数
retryTimes: 3
charset: UTF-8
# 长连接保持时间 单位s,不宜过长
keepAliveTime: 10
# 针对不同的网址,长连接保持的存活时间,单位s,如果是频繁而持续的请求,可以设置小一点,不建议设置过大,避免大量无用连接占用内存资源
keepAliveTargetHost:
www.baidu.com: 5
- maxTotalConnect: 连接池的最大连接数,0代表不限;如果取0,需要考虑连接泄露导致系统崩溃的后果
- maxConnectPerRoute: 每个路由的最大连接数,如果只调用同一个服务端,可以设置和最大连接数相同,也就是一个路由
- connectTimeout: 客户端和服务端建立连接的超时时间,这里最大只能是21s,因为操作系统的tcp进行三次握手时,有它自己的超时时间,即便设置100s也是在21s后报错.
- readTimeout: 也就是socketTime,指的是两个相邻的数据包的间隔超时时间,比如下载一个比较大的文件,就算耗时很长也不会中断,但是如果两次响应时间间隔超过这个值就会报错.
举个例子:
将这个时间设置为2000ms, 然后分别去请求这两个接口,这两个接口总的等待时间都是3000ms
@GetMapping("/sleep")
public String sleep(@RequestParam(required = false,defaultValue = "3000") Integer mils) {
log.info("sleep:{}",mils);
try {
TimeUnit.MILLISECONDS.sleep(mils);
} catch (InterruptedException e) {
e.printStackTrace();
}
return "hello!" + mils;
}
@GetMapping("/sleep/on_and_off")
public void sleepOnAndOff(HttpServletResponse response) {
log.info("sleepOnAndOff");
for (int i = 0;i < 10;i++){
try {
response.getWriter().println("" + i);
response.flushBuffer();
Thread.sleep(300);
} catch (Exception e) {
e.printStackTrace();
}
}
}
调用sleep报读取超时了
调用第二个接口正常返回了
- connectionRequestTimout: 从连接池获取连接的等待时间,不宜过长
- retryTimes: 重试次数
- keepAliveTime: 长连接保持时间, http1.1都是默认开启长连接的,如果不配置这个时间,连接池会默认永久保持连接,这显然是不合理的. 对于需要频繁调用的服务端,我们可以开启长连接,然后将这个保持时间设置小一些,能保证相邻两次请求都长连接都还在就可以了. 并不是越久越好,如果是不频繁访问的服务端,长期保持一个无用的连接也会大大占用资源.
- keepAliveTargetHost: 针对不同的请求地址,可以单独设置不同的长连接存活时间
4. 使用
使用的话还是restTemplate的那一套接口
ResponseEntity<String> entity = httpClientTemplate.getForEntity(url, String.class);
application.yml的日志等级打开debug,可以详细看到参数的配置情况
logging:
level:
ROOT: debug
org.apache.http: debug