增强网关
目的
-
整合错误码,对外显示友好,对内便于快速定位问题
-
记录出错请求,依照错误码制定处理策略
设计
状态码格式
- 示例 E01001B002
- 解析
- E 统一前缀,表明异常
- 01 应用标识
- 001 功能域
- B 错误类型
- 002 错误码
配置格式
核心实现
配置文件读取
- NacosClient 封装后的Nacos客户端,便于手动操作获取配置。
- NacosConfChangeHandler Nacos监听器,用于监听主配置文件的改动。
- CoreContainerService 配置实体容器操作接口
public interface CoreContainerService {
/**
* 数据初始化,用于第一次加载数据
*
* @return 变化的dataId 的集合
*/
void initData();
/**
* 查找服务商管理配置中并且更新服务提供商的列表 那些配置发生了变化,返回 dataId;
*
* @param ServiceProviderList
* @return 返回服务提供商配置列表中变化的dataId
*/
List<String> findServerProviderDataDifferent(List<ServiceProvider> ServiceProviderList);
/**
* 得到服务提供商
*
* @return 得到所有的服务提供商
*/
Map<String, ServiceProvider> getServiceProvider();
/**
* 得到错误码
*
* @return 得到所有的状态码
*/
Map<String, List<UrlStatusCode>> getUrlStatusCode();
/**
* 处理 变化的dataId,并且通知给数据格式转换接口实现类
*
* @param dataDifferent
*/
void fetchUrlStatusCode(List<String> dataDifferent);
}
- DataFormatConversion 数据格式转换接口
public interface DataFormatConversion {
/**
* 更新服务提供商
*
* @param serviceProviderList
*/
void updateServiceProvider(JSONArray serviceProviderList);
/**
* 更新状态码
*
* @param dataId
* @param statusCodeList
*/
void updateStatusCode(String dataId, JSONArray statusCodeList);
}
错误码转换
- CacheRequestBodyFilter 缓存请求体过滤器,获取请求体并重写getBody方法,以便后续过滤器可以多次读取请求参数
@Component
public class CacheRequestBodyFilter implements GlobalFilter, Ordered {
@Override
public Mono<Void> filter(ServerWebExchange exchange, GatewayFilterChain chain) {
MediaType contentType = exchange.getRequest().getHeaders().getContentType();
exchange.getAttributes().put(Constant.Attributes.REQUEST_URL, exchange.getRequest().getURI());
if (exchange.getRequest().getHeaders().getContentType() == null) {
return chain.filter(exchange);
} else {
if (!MediaType.APPLICATION_FORM_URLENCODED.equals(contentType) && !MediaType.APPLICATION_JSON.equals(contentType)) {
return chain.filter(exchange);
}
NettyDataBufferFactory factory = (NettyDataBufferFactory) exchange.getResponse().bufferFactory();
return DataBufferUtils.join(exchange.getRequest().getBody().defaultIfEmpty(factory.wrap(new EmptyByteBuf(factory.getByteBufAllocator()))))
.flatMap(dataBuffer -> {
DataBufferUtils.retain(dataBuffer);
Flux<DataBuffer> cachedFlux = Flux.defer(() -> Flux.just(dataBuffer.slice(0, dataBuffer.readableByteCount())));
ServerHttpRequest mutatedRequest = new ServerHttpRequestDecorator(exchange.getRequest()) {
@Override
public Flux<DataBuffer> getBody() {
return cachedFlux;
}
};
return chain.filter(exchange.mutate().request(mutatedRequest).build());
});
}
}
@Override
public int getOrder() {
return Ordered.HIGHEST_PRECEDENCE;
}
}
- ErrorCodeFilter 错误码转换过滤器
@Component
public class ErrorCodeFilter implements GlobalFilter, Ordered {
private static final Logger logger = LoggerFactory.getLogger(ErrorCodeFilter.class);
private static final String UNKNOWN_ERROR_APP_CODE = "E99999P999";
private static final String UNKNOWN_ERROR_TIPS = "【E99999P999】未知的错误发生了,请联系平台技术人员";
private static final String ERROR_CODE_PLACEHOLDER = "#APPCODE#";
private static final String RAW_MSG_PLACEHOLDER = "#MSG#";
@Autowired
private RequestFailSolve requestFailSolve;
@Override
public Mono<Void> filter(ServerWebExchange exchange, GatewayFilterChain chain) {
ServerHttpRequest serverHttpRequest = exchange.getRequest();
ServerHttpResponse originalResponse = exchange.getResponse();
DataBufferFactory bufferFactory = originalResponse.bufferFactory();
ServerHttpResponseDecorator decoratedResponse = new ServerHttpResponseDecorator(originalResponse) {
@Override
public Mono<Void> writeWith(Publisher<? extends DataBuffer> body) {
String contentType = getDelegate().getHeaders().getFirst(HttpHeaders.CONTENT_TYPE);
List<String> encodings = exchange.getResponse().getHeaders().get(HttpHeaders.CONTENT_ENCODING);
boolean jsonFlag = MediaType.APPLICATION_JSON_VALUE.equals(contentType) || MediaType.APPLICATION_JSON_UTF8_VALUE.equals(contentType);
boolean gzipFlag = encodings != null && encodings.contains("gzip");
if (body instanceof Flux && jsonFlag) {
Flux<? extends DataBuffer> fluxBody = (Flux<? extends DataBuffer>) body;
return super.writeWith(fluxBody.buffer().map(dataBuffer -> {
DataBufferFactory dataBufferFactory = new DefaultDataBufferFactory();
DataBuffer join = dataBufferFactory.join(dataBuffer);
byte[] responseContent = new byte[join.readableByteCount()];
join.read(responseContent);
DataBufferUtils.release(join);
return bufferFactory.wrap(responseAnalysis(exchange, responseContent, gzipFlag));
}));
}
return super.writeWith(body);
}
};
return chain.filter(exchange.mutate().response(decoratedResponse).build());
}
@Override
public int getOrder() {
return NettyWriteResponseFilter.WRITE_RESPONSE_FILTER_ORDER - 20;
}
private byte[] responseAnalysis(ServerWebExchange exchange, byte[] source, boolean gzipFlag) {
// 读取响应内容
if (gzipFlag) {
source = GZIPUtil.uncompress(source);
}
String body = new String(source, StandardCharsets.UTF_8);
// 处理
JSONObject result = JSONObject.parseObject(body);
if (!ResultCode.SUCCESS.getCode().equals(result.getInteger("code"))) {
String responseBody = body;
// 记录请求日志
HttpMethod method = exchange.getRequest().getMethod();
MediaType contentType = exchange.getRequest().getHeaders().getContentType();
if (HttpMethod.POST.equals(method)) {
if (MediaType.APPLICATION_FORM_URLENCODED.equals(contentType) || MediaType.APPLICATION_JSON.equals(contentType)) {
Flux<DataBuffer> requestBody = exchange.getRequest().getBody();
requestBody.subscribe(buffer -> {
CharBuffer charBuffer = StandardCharsets.UTF_8.decode(buffer.asByteBuffer());
DataBufferUtils.release(buffer);
logger.info("请求出现异常!请求Url [{}] 请求头 [{}] 请求内容 [{}] 响应内容 [{}]",
exchange.getRequest().getURI(), exchange.getRequest().getHeaders(), charBuffer.toString(), responseBody);
});
} else {
logger.info("请求出现异常!请求Url [{}] 请求头 [{}] 请求内容为文件 响应内容 [{}]",
exchange.getRequest().getURI(), exchange.getRequest().getHeaders(), responseBody);
}
}
body = errorCodeRevert(result, exchange);
}
// 封装响应内容
byte[] target = new String(body.getBytes(), StandardCharsets.UTF_8).getBytes();
if (gzipFlag) {
target = GZIPUtil.compress(target);
}
return target;
}
private String errorCodeRevert(JSONObject result, ServerWebExchange exchange) {
// 转换响应码
String key = result.getString("path") + "#" + result.getString("appCode");
ErrorCode errorCode = Optional.ofNullable(ErrorCodeDataFormatConversionImpl.errorCodeMap.get(key)).orElse(new ErrorCode());
ServiceProvider provider = ErrorCodeDataFormatConversionImpl.serviceProviderMap.get(errorCode.getDataId());
String code = UNKNOWN_ERROR_APP_CODE;
String msg = UNKNOWN_ERROR_TIPS;
if (provider != null) {
code = String.format("E%s%s%s%s", provider.getCode(), errorCode.getFeatCode(), errorCode.getType(), errorCode.getCode());
msg = errorCode.getTips().replace(ERROR_CODE_PLACEHOLDER, code)
.replace(RAW_MSG_PLACEHOLDER, result.getString("msg"));
}
result.put("appCode", code);
result.put("result", msg);
// 异常策略处理
return requestFailSolve.solveRequestFail(exchange, result, errorCode);
}
}
异常策略处理
- RequestFailSolve 异常请求处理类
- CompensationService 补偿策略接口
使用中的问题
堆外内存泄漏
- 现象
日志报错:
[WARN] 2021-01-21 15:56:34.350 [reactor-http-nio-2] io.netty.channel.AbstractChannelHandlerContext - An exception 'io.netty.util.internal.OutOfDirectMemoryError: failed to allocate 16777216 byte(s) of direct memory (used: 3758096391, max: 3765960704)' [enable DEBUG level for full stacktrace] was thrown by a user handler's exceptionCaught() method while handling the following exception:
io.netty.util.internal.OutOfDirectMemoryError: failed to allocate 16777216 byte(s) of direct memory (used: 3758096391, max: 3765960704)
- 定位问题
开启Netty内存泄漏检测
-Dio.netty.leakDetection.level=PAPANOID
Netty的内存泄漏检测分为四个级别:
- DISABLED - 完成禁止检测内存泄漏,这个是不推荐。
- SIMPLE - 如果buffer中出现1%的内存泄漏,打印错误日志,就是上面那样。但是日志是看不到详细在什么位置出现了内存泄漏。鉴于性能考虑,这个是在生产环境中默认使用。
- ADVANCED - 如果buffer的1%出现内存泄漏,打印错误,并且输出详细的内存泄漏信息。
- PARANOID - 这个和ADVANCED输出的内容是一样的,但它会对每一次请求的buffer做检查。很适用于调试和单元测试。
确认 HttpServerRequestDecoder 时发生了内存泄漏,猜测问题出现在缓存请求体过滤器:
[ERROR] 2021-01-21 16:03:14.726 [reactor-http-nio-8] io.netty.util.ResourceLeakDetector - LEAK: ByteBuf.release() was not called before it's garbage-collected. See http://netty.io/wiki/reference-counted-objects.html for more information.
Recent access records:
Created at:
io.netty.buffer.SimpleLeakAwareByteBuf.unwrappedDerived(SimpleLeakAwareByteBuf.java:143)
io.netty.buffer.SimpleLeakAwareByteBuf.readRetainedSlice(SimpleLeakAwareByteBuf.java:67)
io.netty.handler.codec.http.HttpObjectDecoder.decode(HttpObjectDecoder.java:305)
io.netty.handler.codec.http.HttpServerCodec$HttpServerRequestDecoder.decode(HttpServerCodec.java:101)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:489)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:428)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:253)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:628)
io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:563)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:480)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:442)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
java.lang.Thread.run(Thread.java:748)
再次调试,定位具体代码:
@Override
public Mono<Void> filter(ServerWebExchange exchange, GatewayFilterChain chain) {
exchange.getAttributes().put(Constant.Attributes.REQUEST_URL, exchange.getRequest().getURI());
if (exchange.getRequest().getHeaders().getContentType() == null) {
return chain.filter(exchange);
} else {
DataBufferFactory dataBufferFactory = new DefaultDataBufferFactory();
return DataBufferUtils.join(exchange.getRequest().getBody().defaultIfEmpty(dataBufferFactory.wrap(new byte[0]))).flatMap(dataBuffer -> {
NettyDataBufferFactory nettyDataBufferFactory = new NettyDataBufferFactory(new PooledByteBufAllocator(true));
NettyDataBuffer nettyDataBuffer = nettyDataBufferFactory.wrap(dataBuffer.asByteBuffer());
DataBufferUtils.retain(nettyDataBuffer);
Flux<DataBuffer> cachedFlux = Flux.defer(() -> Flux.just(nettyDataBuffer));
ServerHttpRequest mutatedRequest = new ServerHttpRequestDecorator(exchange.getRequest()) {
@Override
public Flux<DataBuffer> getBody() {
return cachedFlux;
}
};
return chain.filter(exchange.mutate().request(mutatedRequest).build());
});
}
}
定位后发现存在以下问题:
DataBufferUtils.retain(nettyDataBuffer)
语句将 nettyDataBuffer 对象的引用计数+1,以便后续可以再次订阅。但在错误码过滤器中没有及时释放,导致netty直接内存中一直持有缓存的对象。- 没有对请求体内容进行过滤,缓存文件会耗费大量堆内堆外内存,导致OOM
- 网关自身发生异常时,缓存的对象无法释放。
- 处理办法
- 调整请求头缓存过滤器的逻辑,及时释放引用。
- 判断请求类型,只对
application/x-www-form-urlencoded
和application/json
进行缓存。 - 在网关发生异常时,释放掉缓存的对象。
@Override
public Mono<Void> filter(ServerWebExchange exchange, GatewayFilterChain chain) {
MediaType contentType = exchange.getRequest().getHeaders().getContentType();
if (MediaType.APPLICATION_JSON.equals(contentType) || MediaType.APPLICATION_FORM_URLENCODED.equals(contentType)) {
DataBuffer body = exchange.getAttributeOrDefault(CACHED_REQUEST_BODY_ATTR, null);
if (body != null) {
return chain.filter(exchange);
}
return ServerWebExchangeUtils.cacheRequestBody(exchange, (serverHttpRequest) -> {
// don't mutate and build if same request object
if (serverHttpRequest == exchange.getRequest()) {
return chain.filter(exchange);
}
return chain.filter(exchange.mutate().request(serverHttpRequest).build()).doOnError(throwable -> {
log.info("发生异常:异常信息为:", throwable);
Object attribute = exchange.getAttributes().remove(CACHED_REQUEST_BODY_ATTR);
if (attribute != null && attribute instanceof PooledDataBuffer) {
PooledDataBuffer dataBuffer = (PooledDataBuffer) attribute;
if (dataBuffer.isAllocated()) {
dataBuffer.release();
}
}
});
});
}
return chain.filter(exchange);
}
public class GlobalExceptionHandler extends DefaultErrorWebExceptionHandler {
public GlobalExceptionHandler(ErrorAttributes errorAttributes, ResourceProperties resourceProperties,
ErrorProperties errorProperties,
ApplicationContext applicationContext) {
super(errorAttributes, resourceProperties, errorProperties, applicationContext);
}
//省略无关代码
@Override
protected Map<String, Object> getErrorAttributes(ServerRequest request, ErrorAttributeOptions options) {
log.info("发生异常:异常信息为:{}", request.attributes().get(ERROR_ATTRIBUTE));
//释放缓存的body
Object attribute = request.attributes().remove("cachedRequestBody");
if (attribute != null && attribute instanceof PooledDataBuffer) {
PooledDataBuffer dataBuffer = (PooledDataBuffer) attribute;
if (dataBuffer.isAllocated()) {
dataBuffer.release();
}
}
//重写响应体
return response(1000);
}
}