gateway的增强网关设计与使用

背景

明文参数在网络传输中会被抓包，即使加了https，所以需要对参数加密进行密文传输，采用gateway网关需要对请求进来的参数进行解密，然后再分发到对应的服务。这里就需要涉及到修改body内容。

设计思路

DynamicRouteServiceListener

/**
 * 动态路由监听器
 *
 */
@Order
@Slf4j
@Component
public class DynamicRouteServiceListener {

	private final DynamicRouteService dynamicRouteService;
	private final NacosDiscoveryProperties nacosDiscoveryProperties;
	private final NacosConfigProperties nacosConfigProperties;
	private final RenegadeProperties renegadeProperties;

	public DynamicRouteServiceListener(DynamicRouteService dynamicRouteService, NacosDiscoveryProperties nacosDiscoveryProperties, NacosConfigProperties nacosConfigProperties, RenegadeProperties renegadeProperties) {
		this.dynamicRouteService = dynamicRouteService;
		this.nacosDiscoveryProperties = nacosDiscoveryProperties;
		this.nacosConfigProperties = nacosConfigProperties;
		this.renegadeProperties= renegadeProperties;
		dynamicRouteServiceListener();
	}

	/**
	 * 监听Nacos下发的动态路由配置
	 */
	private void dynamicRouteServiceListener() {
		try {
			String dataId = NacosConstant.dataId(renegadeProperties.getName(), renegadeProperties.getEnv(), NacosConstant.NACOS_CONFIG_JSON_FORMAT);
			String group = nacosConfigProperties.getGroup();
			Properties properties = new Properties();
			properties.setProperty(PropertyKeyConst.SERVER_ADDR, nacosDiscoveryProperties.getServerAddr());
			properties.setProperty(PropertyKeyConst.NAMESPACE, nacosDiscoveryProperties.getNamespace());
			ConfigService configService = NacosFactory.createConfigService(properties);
			configService.addListener(dataId, group, new Listener() {
				@Override
				public void receiveConfigInfo(String configInfo) {
					List<RouteDefinition> routeDefinitions = JSON.parseArray(configInfo, RouteDefinition.class);
					dynamicRouteService.updateList(routeDefinitions);
				}

				@Override
				public Executor getExecutor() {
					return null;
				}
			});
			String configInfo = configService.getConfig(dataId, group, 5000);
			if (configInfo != null) {
				List<RouteDefinition> routeDefinitions = JSON.parseArray(configInfo, RouteDefinition.class);
				dynamicRouteService.updateList(routeDefinitions);
			}
		} catch (NacosException ignored) {

		}
	}

}

ErrorExceptionHandler

/**
 * 异常处理 后续自定义异常可在处理
 *
 * 
 */
public class ErrorExceptionHandler extends DefaultErrorWebExceptionHandler {

	public ErrorExceptionHandler(ErrorAttributes errorAttributes, ResourceProperties resourceProperties,
								 ErrorProperties errorProperties, ApplicationContext applicationContext) {
		super(errorAttributes, resourceProperties, errorProperties, applicationContext);
	}

	/**
	 * 获取异常属性
	 */
	@Override
	protected Map<String, Object> getErrorAttributes(ServerRequest request, boolean includeStackTrace) {
		int code = 500;
		Throwable error = super.getError(request);
		if (error instanceof NotFoundException) {
			code = 404;
		}
		if (error instanceof ResponseStatusException) {
			code = ((ResponseStatusException) error).getStatus().value();
		}
		//释放缓存的body，防止OOM
		Object attribute = request.attributes().remove(CACHED_REQUEST_BODY_ATTR);
		if (attribute instanceof PooledDataBuffer) {
			PooledDataBuffer dataBuffer = (PooledDataBuffer) attribute;
			if (dataBuffer.isAllocated()) {
				dataBuffer.release();
			}
		}
		return ResponseProvider.response(code, this.buildMessage(request, error));
	}

	/**
	 * 指定响应处理方法为JSON处理的方法
	 *
	 * @param errorAttributes
	 */
	@Override
	protected RouterFunction<ServerResponse> getRoutingFunction(ErrorAttributes errorAttributes) {
		return RouterFunctions.route(RequestPredicates.all(), this::renderErrorResponse);
	}

	/**
	 * 根据code获取对应的HttpStatus
	 *
	 * @param errorAttributes
	 */
	@Override
	protected HttpStatus getHttpStatus(Map<String, Object> errorAttributes) {
		int statusCode = (int) errorAttributes.get("code");
		return HttpStatus.valueOf(statusCode);
	}

	/**
	 * 构建异常信息
	 *
	 * @param request
	 * @param ex
	 * @return
	 */
	private String buildMessage(ServerRequest request, Throwable ex) {
		StringBuilder message = new StringBuilder("Failed to handle request [");
		message.append(request.methodName());
		message.append(" ");
		message.append(request.uri());
		message.append("]");
		if (ex != null) {
			message.append(": ");
			message.append(ex.getMessage());
		}
		return message.toString();
	}

}

CacheBodyGlobalFilter

缓存body，否则只能被拿一次，后续就拿不到了。

public class CacheBodyGlobalFilter implements Ordered, GlobalFilter {
	@Override
	public Mono<Void> filter(ServerWebExchange exchange, GatewayFilterChain chain) {
		MediaType contentType = exchange.getRequest().getHeaders().getContentType();
		//缓存body的类型限定
		String encryptionKey = exchange.getRequest().getHeaders().getFirst("encryptionKey");
		if (!StringUtils.isEmpty(encryptionKey)) {
			if (MediaType.APPLICATION_JSON.equals(contentType) || MediaType.APPLICATION_FORM_URLENCODED.equals(contentType)) {
				return DataBufferUtils.join(exchange.getRequest().getBody())
					.flatMap(dataBuffer -> {
					//缓存关键点，采用netty里的引用计数器，让下一个计数器引用
						DataBufferUtils.retain(dataBuffer);
						Flux<DataBuffer> cachedFlux = Flux
							.defer(() -> Flux.just(dataBuffer.slice(0, dataBuffer.readableByteCount())));
						ServerHttpRequest mutatedRequest = new ServerHttpRequestDecorator(
							exchange.getRequest()) {
							@Override
							public Flux<DataBuffer> getBody() {
								return cachedFlux;
							}
						};
						//防止oom 一定需要释放
						return chain.filter(exchange.mutate().request(mutatedRequest).build()).doOnError(throwable -> {
							log.info("发生异常:异常信息为:", throwable);
							Object attribute = exchange.getAttributes().remove(CACHED_REQUEST_BODY_ATTR);
							if (attribute instanceof PooledDataBuffer) {
								PooledDataBuffer pooledDataBuffer = (PooledDataBuffer) attribute;
								if (pooledDataBuffer.isAllocated()) {
									pooledDataBuffer.release();
								}
							}
						});
					});
			}
		} else {
			return chain.filter(exchange);
		}
		return chain.filter(exchange);
	}

	@Override
	public int getOrder() {
		//级别需要最高的
		return Ordered.HIGHEST_PRECEDENCE;
	}
}

RequestFilter

解密以及路由处理过滤器

/**
 * <p>
 * 全局拦截器，作用所有的微服务
 * <p>
 * 1. 对请求头中参数进行处理 from 参数进行清洗
 * 2. 重写StripPrefix = 1,支持全局
 *
 *
 */
@Component
@Slf4j
@AllArgsConstructor
public class RequestFilter implements GlobalFilter, Ordered {
	private AuthProperties authProperties;
	private ObjectMapper objectMapper;

	/**
	 * Process the Web request and (optionally) delegate to the next
	 * {@code WebFilter} through the given {@link GatewayFilterChain}.
	 *
	 * @param exchange the current server exchange
	 * @param chain    provides a way to delegate to the next filter
	 * @return {@code Mono<Void>} to indicate when request processing is complete
	 */
	@SneakyThrows
	@Override
	public Mono<Void> filter(ServerWebExchange exchange, GatewayFilterChain chain) {
		// 1. 清洗请求头中from 参数
		ServerHttpRequest request = exchange.getRequest().mutate()
			.headers(httpHeaders -> httpHeaders.remove("X"))
			.build();

		// 2. 重写StripPrefix
		addOriginalRequestUrl(exchange, request.getURI());
		String rawPath = request.getURI().getRawPath();
		String newPath = "/" + Arrays.stream(StringUtils.tokenizeToStringArray(rawPath, "/"))
			.skip(1L).collect(Collectors.joining("/"));
		ServerHttpRequest newRequest = null;
		//后续参数进行加签加密等一系列的处理，可以在这里进行拓展
		String encryptionKey = request.getHeaders().getFirst("encryptionKey");
		//2021-10-21 encryptionKey不为空，代表参数需要解密，本系统采用aes 128位配合rsa的1024位混合加密，对参数进行脱敏
		if (!StringUtils.isEmpty(encryptionKey)) {
			log.debug(" encrypt aes  key{}", encryptionKey);
			HttpMethod method = request.getMethod();
			Flux<DataBuffer> bodys = request.getBody();
			String body = resolveBodyFromRequest(bodys);
			if (method == HttpMethod.POST || method == HttpMethod.PUT) {
				if (MediaType.APPLICATION_FORM_URLENCODED.equals(request.getHeaders().getContentType())) {
					// 解密rsa加密过的aes的私钥
					newPath = newPath + "?" + encryptBody(encryptionKey, body);
					newRequest = request.mutate().path(newPath).build();
				} else {

					//从请求里获取Post请求体
					log.debug("body body aes  key{}", body);
					if (body != null) {
						// 解密rsa加密过的aes的私钥
						String context = encryptBody(encryptionKey, body);
						DataBuffer bdyDataBuffer = stringBuffer(context);
						// //封装request，传给下一级
						newRequest = request.mutate().path(newPath).build();
						Flux<DataBuffer> bodyFlux = Flux.just(bdyDataBuffer);
						newRequest = new ServerHttpRequestDecorator(newRequest) {
							@Override
							public Flux<DataBuffer> getBody() {
								return bodyFlux;
							}
						};
						HttpHeaders headers = new HttpHeaders();
						headers.putAll(exchange.getRequest().getHeaders());
						//坑逼：：： 由于修改了传递参数，需要重新设置CONTENT_LENGTH，长度是字节长度，不是字符串长度,如果不设置，body获取不到
						int length = context.getBytes().length;
						headers.remove(HttpHeaders.CONTENT_LENGTH);
						headers.setContentLength(length);
						newRequest = new ServerHttpRequestDecorator(newRequest) {
							@Override
							public HttpHeaders getHeaders() {
								return headers;
							}
						};
					}
				}
			}
		} else {
			//未塞入key，但是URL再这里面的，需要加密，那么就直接return
			if (isAnyMatch(newPath)) {
				return unEncrypt(exchange.getResponse(), "URL参数缺失");
			}
			newRequest = request.mutate().path(newPath).build();
		}
		exchange.getAttributes().put(GATEWAY_REQUEST_URL_ATTR, newRequest.getURI());
		return chain.filter(exchange.mutate().request(newRequest.mutate().build()).build()).doOnError(throwable -> {
			log.info("发生异常:异常信息为:", throwable);
			Object attribute = exchange.getAttributes().remove(CACHED_REQUEST_BODY_ATTR);
			if (attribute instanceof PooledDataBuffer) {
				PooledDataBuffer pooledDataBuffer = (PooledDataBuffer) attribute;
				if (pooledDataBuffer.isAllocated()) {
					pooledDataBuffer.release();
				}
			}
		});
	}

	private String encryptBody(String encryptionKey, String body) throws Exception {
		byte[] hexStringToBytes = Base64.decode(encryptionKey);
		byte[] decryptByPrivateKey = RSAUtilApp.decryptByPrivateKey(hexStringToBytes,
			SysSecurityKeyConstant.privateKey_app);
		log.info(" decrypt aes key {}", new String(decryptByPrivateKey, StandardCharsets.UTF_8));
		String context = AESUtil.aesDecrypt(body, new String(decryptByPrivateKey, StandardCharsets.UTF_8));
		log.info("decrypt request param {}", context);
		return context;
	}

	private String resolveBodyFromRequest(Flux<DataBuffer> body) {
		//获取请求体
		AtomicReference<String> bodyRef = new AtomicReference<>();
		body.subscribe(buffer -> {
			CharBuffer charBuffer = StandardCharsets.UTF_8.decode(buffer.asByteBuffer());
			DataBufferUtils.release(buffer);
			bodyRef.set(charBuffer.toString());
		});
		//获取request body
		return bodyRef.get();
	}


	/**
	 * 用netty封装贼猛
	 *
	 * @param value
	 * @return
	 */
	private DataBuffer stringBuffer(String value) {
		byte[] bytes = value.getBytes(StandardCharsets.UTF_8);
		NettyDataBufferFactory nettyDataBufferFactory = new NettyDataBufferFactory(ByteBufAllocator.DEFAULT);
		DataBuffer buffer = nettyDataBufferFactory.allocateBuffer(bytes.length);
		buffer.write(bytes);
		return buffer;
	}

	/**
	 * 符合的才加密
	 *
	 * @param path
	 * @return
	 */
	private boolean isAnyMatch(String path) {
		return authProperties.getEncryptUrl().stream().anyMatch(path:: equals);
	}

	private Mono<Void> unEncrypt(ServerHttpResponse resp, String msg) {
		resp.setStatusCode(HttpStatus.FORBIDDEN);
		return getRep(resp, msg, objectMapper, log);
	}

	static Mono<Void> getRep(ServerHttpResponse resp, String msg, ObjectMapper objectMapper, Logger log) {
		resp.getHeaders().add("Content-Type", "application/json;charset=UTF-8");
		String result = "";
		try {
			result = objectMapper.writeValueAsString(ResponseProvider.unAuth(msg));
		} catch (JsonProcessingException e) {
			log.error(e.getMessage(), e);
		}
		DataBuffer buffer = resp.bufferFactory().wrap(result.getBytes(StandardCharsets.UTF_8));
		return resp.writeWith(Flux.just(buffer));
	}

	@Override
	public int getOrder() {
		return -1000;
	}

}

使用中的问题

• bodoy读取不全问题
  网上说只能读取部分，利用缓存方式，60000个没问题，满足我的业务需求，由于加密的串不会超过string最大长度。

• 堆外内存泄漏

• 开启Netty内存泄漏检测

-Dio.netty.leakDetection.level=PAPANOID

Netty的内存泄漏检测分为四个级别：
DISABLED - 完成禁止检测内存泄漏，这个是不推荐。 SIMPLE - 如果buffer中出现1%的内存泄漏，打印错误日志，就是上面那样。但是日志是看不到详细在什么位置出现了内存泄漏。鉴于性能考虑，这个是在生产环境中默认使用。 ADVANCED - 如果buffer的1%出现内存泄漏，打印错误，并且输出详细的内存泄漏信息。 PARANOID - 这个和ADVANCED输出的内容是一样的，但它会对每一次请求的buffer做检查。很适用于调试和单元测试。

  [reactor-http-nio-8] io.netty.util.ResourceLeakDetector - LEAK: ByteBuf.release() was not called before it's garbage-collected. See http://netty.io/wiki/reference-counted-objects.html for more information.
Recent access records: 
Created at:
	io.netty.buffer.SimpleLeakAwareByteBuf.unwrappedDerived(SimpleLeakAwareByteBuf.java:143)
	io.netty.buffer.SimpleLeakAwareByteBuf.readRetainedSlice(SimpleLeakAwareByteBuf.java:67)

• 定位后发现存在以下问题：

DataBufferUtils.retain(nettyDataBuffer) 语句将 nettyDataBuffer 对象的引用计数+1，以便后续可以再次订阅。但在错误码过滤器中没有及时释放，导致netty直接内存中一直持有缓存的对象。
没有对请求体内容进行过滤，缓存文件会耗费大量堆内堆外内存，导致OOM
网关自身发生异常时，缓存的对象无法释放。
处理办法
调整请求头缓存过滤器的逻辑，及时释放引用。
判断请求类型，只对 application/x-www-form-urlencoded 和 application/json进行缓存。
在网关发生异常时，释放掉缓存的对象。

• 解决方案
  在发生异常的时候，手动释放掉内存
  关键语句

.doOnError(throwable -> {
			log.info("发生异常:异常信息为:", throwable);
			Object attribute = exchange.getAttributes().remove(CACHED_REQUEST_BODY_ATTR);
			if (attribute instanceof PooledDataBuffer) {
				PooledDataBuffer pooledDataBuffer = (PooledDataBuffer) attribute;
				if (pooledDataBuffer.isAllocated()) {
					pooledDataBuffer.release();
				}
			}
		});
	//释放缓存的body，防止OOM
		Object attribute = request.attributes().remove(CACHED_REQUEST_BODY_ATTR);
		if (attribute instanceof PooledDataBuffer) {
			PooledDataBuffer dataBuffer = (PooledDataBuffer) attribute;
			if (dataBuffer.isAllocated()) {
				dataBuffer.release();
			}
		}