RestTemplate负载均衡过程

RestTemplate如何使用负载均衡过程

使用方式

通常会有如下使用，加上@LoadBalanced注解使restTemplate具有负载均衡能力

@LoadBalanced
@Bean
RestTemplate restTemplate() {
    return new RestTemplate();
}

先看一下LoadBalanced注解

@Target({ ElementType.FIELD, ElementType.PARAMETER, ElementType.METHOD })
@Retention(RetentionPolicy.RUNTIME)
@Documented
@Inherited
@Qualifier
public @interface LoadBalanced {
}

看到Qualifier这个注解大家应该就明白LoadBalanced注解的原理了。

负载均衡过程

本文主要了解下restTemplate如何结合负载均衡的调用过程，通过restTemplate其中一个方法来分析，如下

public <T> ResponseEntity<T> exchange(String url, HttpMethod method,
		@Nullable HttpEntity<?> requestEntity, Class<T> responseType, Object... uriVariables)
		throws RestClientException {

	RequestCallback requestCallback = httpEntityCallback(requestEntity, responseType);
	ResponseExtractor<ResponseEntity<T>> responseExtractor = responseEntityExtractor(responseType);
    // ①execute方法
	return nonNull(execute(url, method, requestCallback, responseExtractor, uriVariables));
}

public <T> T execute(String url, HttpMethod method, @Nullable RequestCallback requestCallback,
		@Nullable ResponseExtractor<T> responseExtractor, Object... uriVariables) throws RestClientException {

	URI expanded = getUriTemplateHandler().expand(url, uriVariables);
    // ②doExecute方法
	return doExecute(expanded, method, requestCallback, responseExtractor);
}

protected <T> T doExecute(URI url, @Nullable HttpMethod method, @Nullable RequestCallback requestCallback,
		@Nullable ResponseExtractor<T> responseExtractor) throws RestClientException {

	Assert.notNull(url, "URI is required");
	Assert.notNull(method, "HttpMethod is required");
	ClientHttpResponse response = null;
	try {
	    // ③跟进createRequest方法，request为InterceptingClientHttpRequest实例
        // InterceptingClientHttpRequest继承AbstractBufferingClientHttpRequest继承AbstractClientHttpRequest
		ClientHttpRequest request = createRequest(url, method);
    if (requestCallback != null) {
          // requestCallback为RestTemplate.HttpEntityRequestCallback类
          // 主要是填充一些request属性
	      requestCallback.doWithRequest(request);
    }
    // 执行调用逻辑，实际会调用request的父类方法AbstractClientHttpRequest
    response = request.execute();
    // 省略其他内容
}

HttpAccessor.java

protected ClientHttpRequest createRequest(URI url, HttpMethod method) throws IOException {
    // ④跟进此处的getRequestFactory()和createRequest方法
	ClientHttpRequest request = getRequestFactory().createRequest(url, method);
    // 调用initialize方法，如下
	initialize(request);
	if (logger.isDebugEnabled()) {
		logger.debug("HTTP " + method.name() + " " + url);
	}
	return request;
}

private void initialize(ClientHttpRequest request) {
    // 看下clientHttpRequestInitializers哪来的
	this.clientHttpRequestInitializers.forEach(initializer -> initializer.initialize(request));
}

InterceptingHttpAccessor.java

getRequestFactory()方法

public ClientHttpRequestFactory getRequestFactory() {
    // ⑥getInterceptors()方法后面讲
	List<ClientHttpRequestInterceptor> interceptors = getInterceptors();
	if (!CollectionUtils.isEmpty(interceptors)) {
        // TODO 看下该属性来源，此处类型：InterceptingClientHttpRequestFactory
		ClientHttpRequestFactory factory = this.interceptingRequestFactory;
		if (factory == null) {
			factory = new InterceptingClientHttpRequestFactory(super.getRequestFactory(), interceptors);
			this.interceptingRequestFactory = factory;
		}
		return factory;
	}
	else {
		return super.getRequestFactory();
	}
}

AbstractClientHttpRequestFactoryWrapper.java

createRequest方法

public final ClientHttpRequest createRequest(URI uri, HttpMethod httpMethod) throws IOException {
    // this.requestFactory此属性应该是子类设置的
	return createRequest(uri, httpMethod, this.requestFactory);
}

InterceptingClientHttpRequestFactory.java

protected ClientHttpRequest createRequest(URI uri, HttpMethod httpMethod, ClientHttpRequestFactory requestFactory) {
	return new InterceptingClientHttpRequest(requestFactory, this.interceptors, uri, httpMethod);
}

AbstractClientHttpRequest.java

request.execute()的调用过程

public final ClientHttpResponse execute() throws IOException {
	assertNotExecuted();
    // 调用子类AbstractBufferingClientHttpRequest的实现方法
	ClientHttpResponse result = executeInternal(this.headers);
	this.executed = true;
	return result;
}

AbstractBufferingClientHttpRequest.java

protected ClientHttpResponse executeInternal(HttpHeaders headers) throws IOException {
	byte[] bytes = this.bufferedOutput.toByteArray();
	if (headers.getContentLength() < 0) {
		headers.setContentLength(bytes.length);
	}
    // 子类InterceptingClientHttpRequest的方法
	ClientHttpResponse result = executeInternal(headers, bytes);
	this.bufferedOutput = new ByteArrayOutputStream(0);
	return result;
}

InterceptingClientHttpRequest.java

protected final ClientHttpResponse executeInternal(HttpHeaders headers, byte[] bufferedOutput) throws IOException {
	InterceptingRequestExecution requestExecution = new InterceptingRequestExecution();
    // 当前类的方法
	return requestExecution.execute(this, bufferedOutput);
}

public ClientHttpResponse execute(HttpRequest request, byte[] body) throws IOException {
    // iterator为负载均衡拦截器迭代器，主要是ClientHttpRequestInterceptor的实现类，常见的如LoadBalancerInterceptor、RetryLoadBalancerInterceptor，本例中为RetryLoadBalancerInterceptor，具体用哪一种还要看当前项目的情况，多为springboot的一些condition判读，如以是否有RetryTemplate来判断是否使用RetryLoadBalancerInterceptor
	if (this.iterator.hasNext()) {
		ClientHttpRequestInterceptor nextInterceptor = this.iterator.next();
		return nextInterceptor.intercept(request, body, this);
	}
	else {
		HttpMethod method = request.getMethod();
		Assert.state(method != null, "No standard HTTP method");
		ClientHttpRequest delegate = requestFactory.createRequest(request.getURI(), method);
		request.getHeaders().forEach((key, value) -> delegate.getHeaders().addAll(key, value));
		if (body.length > 0) {
			if (delegate instanceof StreamingHttpOutputMessage) {
				StreamingHttpOutputMessage streamingOutputMessage = (StreamingHttpOutputMessage) delegate;
				streamingOutputMessage.setBody(outputStream -> StreamUtils.copy(body, outputStream));
			}
			else {
				StreamUtils.copy(body, delegate.getBody());
			}
		}
		return delegate.execute();
	}
}

此处看一下RetryLoadBalancerInterceptor的逻辑，RetryLoadBalancerInterceptor如何创建的看后文

public ClientHttpResponse intercept(final HttpRequest request, final byte[] body,
		final ClientHttpRequestExecution execution) throws IOException {
	final URI originalUri = request.getURI();
    // serviceName为负载均衡用
	final String serviceName = originalUri.getHost();
	Assert.state(serviceName != null,
			"Request URI does not contain a valid hostname: " + originalUri);
    // lbRetryFactory为RibbonLoadBalancedRetryFactory
    // loadBalancer为RibbonLoadBalancerClient
	final LoadBalancedRetryPolicy retryPolicy = this.lbRetryFactory
			.createRetryPolicy(serviceName, this.loadBalancer);
	RetryTemplate template = createRetryTemplate(serviceName, request, retryPolicy);
	return template.execute(context -> {
		ServiceInstance serviceInstance = null;
		if (context instanceof LoadBalancedRetryContext) {
			LoadBalancedRetryContext lbContext = (LoadBalancedRetryContext) context;
			serviceInstance = lbContext.getServiceInstance();
		}
		if (serviceInstance == null) {
            // 选择服务实例
			serviceInstance = this.loadBalancer.choose(serviceName);
		}
		ClientHttpResponse response = RetryLoadBalancerInterceptor.this.loadBalancer
				.execute(serviceName, serviceInstance,
						this.requestFactory.createRequest(request, body, execution));
		int statusCode = response.getRawStatusCode();
		if (retryPolicy != null && retryPolicy.retryableStatusCode(statusCode)) {
			byte[] bodyCopy = StreamUtils.copyToByteArray(response.getBody());
			response.close();
			throw new ClientHttpResponseStatusCodeException(serviceName, response,
					bodyCopy);
		}
		return response;
	}, new LoadBalancedRecoveryCallback<ClientHttpResponse, ClientHttpResponse>() {
		// This is a special case, where both parameters to
		// LoadBalancedRecoveryCallback are
		// the same. In most cases they would be different.
		@Override
		protected ClientHttpResponse createResponse(ClientHttpResponse response,
				URI uri) {
			return response;
		}
	});
}

RibbonLoadBalancedRetryFactory

public LoadBalancedRetryPolicy createRetryPolicy(String service,
		ServiceInstanceChooser serviceInstanceChooser) {
    // clientFactory为SpringClientFactory
    // TODO RibbonLoadBalancerContext的作用
	RibbonLoadBalancerContext lbContext = this.clientFactory
			.getLoadBalancerContext(service);
	return new RibbonLoadBalancedRetryPolicy(service, lbContext,
			serviceInstanceChooser, clientFactory.getClientConfig(service));
}

RibbonLoadBalancerClient.java

public ServiceInstance choose(String serviceId) {
	return choose(serviceId, null);
}

public ServiceInstance choose(String serviceId, Object hint) {
	Server server = getServer(getLoadBalancer(serviceId), hint);
	if (server == null) {
		return null;
	}
	return new RibbonServer(serviceId, server, isSecure(server, serviceId),
			serverIntrospector(serviceId).getMetadata(server));
}

protected ILoadBalancer getLoadBalancer(String serviceId) {
	return this.clientFactory.getLoadBalancer(serviceId);
}

protected Server getServer(ILoadBalancer loadBalancer, Object hint) {
	if (loadBalancer == null) {
		return null;
	}
	// Use 'default' on a null hint, or just pass it on?
	return loadBalancer.chooseServer(hint != null ? hint : "default");
}

SpringClientFactory.java

public ILoadBalancer getLoadBalancer(String name) {
	return getInstance(name, ILoadBalancer.class);
}

ZoneAwareLoadBalancer.java

public Server chooseServer(Object key) {
    if (!ENABLED.get() || getLoadBalancerStats().getAvailableZones().size() <= 1) {
        logger.debug("Zone aware logic disabled or there is only one zone");
        // 调用父类BaseLoadBalancer的方法
        return super.chooseServer(key);
    }
    Server server = null;
    try {
        LoadBalancerStats lbStats = getLoadBalancerStats();
        Map<String, ZoneSnapshot> zoneSnapshot = ZoneAvoidanceRule.createSnapshot(lbStats);
        logger.debug("Zone snapshots: {}", zoneSnapshot);
        if (triggeringLoad == null) {
            triggeringLoad = DynamicPropertyFactory.getInstance().getDoubleProperty(
                    "ZoneAwareNIWSDiscoveryLoadBalancer." + this.getName() + ".triggeringLoadPerServerThreshold", 0.2d);
        }

        if (triggeringBlackoutPercentage == null) {
            triggeringBlackoutPercentage = DynamicPropertyFactory.getInstance().getDoubleProperty(
                    "ZoneAwareNIWSDiscoveryLoadBalancer." + this.getName() + ".avoidZoneWithBlackoutPercetage", 0.99999d);
        }
        Set<String> availableZones = ZoneAvoidanceRule.getAvailableZones(zoneSnapshot, triggeringLoad.get(), triggeringBlackoutPercentage.get());
        logger.debug("Available zones: {}", availableZones);
        if (availableZones != null &&  availableZones.size() < zoneSnapshot.keySet().size()) {
            String zone = ZoneAvoidanceRule.randomChooseZone(zoneSnapshot, availableZones);
            logger.debug("Zone chosen: {}", zone);
            if (zone != null) {
                BaseLoadBalancer zoneLoadBalancer = getLoadBalancer(zone);
                server = zoneLoadBalancer.chooseServer(key);
            }
        }
    } catch (Exception e) {
        logger.error("Error choosing server using zone aware logic for load balancer={}", name, e);
    }
    if (server != null) {
        return server;
    } else {
        logger.debug("Zone avoidance logic is not invoked.");
        return super.chooseServer(key);
    }
}

BaseLoadBalancer.java

public Server chooseServer(Object key) {
    if (counter == null) {
        counter = createCounter();
    }
    counter.increment();
    if (rule == null) {
        return null;
    } else {
        try {
            // rule为ZoneAvoidanceRule或其子类（自定义rule）
            return rule.choose(key);
        } catch (Exception e) {
            logger.warn("LoadBalancer [{}]:  Error choosing server for key {}", name, key, e);
            return null;
        }
    }
}

ZoneAvoidanceRule的choose方法会获取断言信息，通过断言进行过滤具体看下AbstractServerPredicate类的getEligibleServers方法

public List<Server> getEligibleServers(List<Server> servers, Object loadBalancerKey) {
    if (loadBalancerKey == null) {
        return ImmutableList.copyOf(Iterables.filter(servers, this.getServerOnlyPredicate()));            
    } else {
        List<Server> results = Lists.newArrayList();
        // 对各个server进行断言判断，如果符合则放到results里以便下一步选择使用
        for (Server server: servers) {
            // 自定义断言会继承AbstractServerPredicate，然后实现apply方法
            if (this.apply(new PredicateKey(loadBalancerKey, server))) {
                results.add(server);
            }
        }
        return results;            
    }
}

RetryLoadBalancerInterceptor如何创建的

LoadBalancerAutoConfiguration

@Bean
@ConditionalOnMissingBean
public RetryLoadBalancerInterceptor ribbonInterceptor(
		LoadBalancerClient loadBalancerClient,
		LoadBalancerRetryProperties properties,
		LoadBalancerRequestFactory requestFactory,
		LoadBalancedRetryFactory loadBalancedRetryFactory) {
    // loadBalancedRetryFactory为RibbonLoadBalancedRetryFactory
	return new RetryLoadBalancerInterceptor(loadBalancerClient, properties,
			requestFactory, loadBalancedRetryFactory);
}

RibbonAutoConfiguration

@Bean
@ConditionalOnClass(name = "org.springframework.retry.support.RetryTemplate")
@ConditionalOnMissingBean
public LoadBalancedRetryFactory loadBalancedRetryPolicyFactory(
		final SpringClientFactory clientFactory) {
	return new RibbonLoadBalancedRetryFactory(clientFactory);
}