springcloudalibaba+nacos+gateway灰度发布

整理并实践nacos+gateway灰度发布

1.项目结构

2.maven依赖

  <parent>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-parent</artifactId>
        <version>2.2.6.RELEASE</version>
        <relativePath/>
    </parent>
    <dependency>
         <groupId>org.springframework.cloud</groupId>
         <artifactId>spring-cloud-dependencies</artifactId>
         <version>Hoxton.SR9</version>
         <type>pom</type>
         <scope>import</scope>
     </dependency>
     <dependency>
         <groupId>com.alibaba.cloud</groupId>
         <artifactId>spring-cloud-alibaba-dependencies</artifactId>
         <version>2.2.6.RELEASE</version>
         <type>pom</type>
         <scope>import</scope>
     </dependency>

前言

目前Spring Cloud官网已经不推荐使用Ribbon作为负载均衡工具使用进入了维护模式，而是推荐使用Spring Cloud LoadBalancer去代替它，但是由于目前市面上还有很多公司在使用Ribbon，此处做简短介绍。
如果希望使用LoadBalancer，替换步骤：
将spring.cloud.loadbalancer.ribbon.enabledproperty 的值设置为false
依赖

<!--客户端负载均衡loadbalancer-->
 <dependency>
      <groupId>org.springframework.cloud</groupId>
      <artifactId>spring-cloud-starter-loadbalancer</artifactId>
  </dependency>

下面基于loadbalancer的实现逻辑,
默认的是ribbon的负载均衡逻辑，请参考其他资料

1.gateway网关灰度逻辑

1.GrayReactiveLoadBalancerClientFilter

import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cloud.client.ServiceInstance;
import org.springframework.cloud.client.loadbalancer.LoadBalancerUriTools;
import org.springframework.cloud.client.loadbalancer.reactive.DefaultRequest;
import org.springframework.cloud.client.loadbalancer.reactive.Request;
import org.springframework.cloud.client.loadbalancer.reactive.Response;
import org.springframework.cloud.gateway.config.LoadBalancerProperties;
import org.springframework.cloud.gateway.filter.GatewayFilterChain;
import org.springframework.cloud.gateway.filter.ReactiveLoadBalancerClientFilter;
import org.springframework.cloud.gateway.support.DelegatingServiceInstance;
import org.springframework.cloud.gateway.support.NotFoundException;
import org.springframework.cloud.gateway.support.ServerWebExchangeUtils;
import org.springframework.cloud.loadbalancer.core.ServiceInstanceListSupplier;
import org.springframework.cloud.loadbalancer.support.LoadBalancerClientFactory;
import org.springframework.http.HttpHeaders;
import org.springframework.stereotype.Component;
import org.springframework.web.server.ServerWebExchange;
import reactor.core.publisher.Mono;

import java.net.URI;

/**
 * @author admin
 * 目前Spring Cloud官网已经不推荐使用Ribbon作为负载均衡工具使用进入了维护模式，
 * 而是推荐使用Spring Cloud LoadBalancer去代替它，但是由于目前市面上还有很多公司在使用Ribbon，此处做简短介绍。
 * 如果希望使用LoadBalancer，替换步骤：
 * 将spring.cloud.loadbalancer.ribbon.enabledproperty 的值设置为false
 */
@Slf4j
@Component
public class GrayReactiveLoadBalancerClientFilter extends ReactiveLoadBalancerClientFilter {

    @Autowired
    private ServerGrayProperty serverGrayProperty;

    private final LoadBalancerClientFactory grayClientFactory;

    public GrayReactiveLoadBalancerClientFilter(LoadBalancerClientFactory clientFactory, LoadBalancerProperties properties) {
        super(clientFactory, properties);
        this.grayClientFactory = clientFactory;
    }


    @Override
    public Mono<Void> filter(ServerWebExchange exchange, GatewayFilterChain chain) {
        URI url = (URI) exchange.getAttribute(ServerWebExchangeUtils.GATEWAY_REQUEST_URL_ATTR);
        String schemePrefix = (String) exchange.getAttribute(ServerWebExchangeUtils.GATEWAY_SCHEME_PREFIX_ATTR);
        if (url != null && ("lb".equals(url.getScheme()) || "lb".equals(schemePrefix))) {
            ServerWebExchangeUtils.addOriginalRequestUrl(exchange, url);
            if (log.isTraceEnabled()) {
                log.trace(ReactiveLoadBalancerClientFilter.class.getSimpleName() + " url before: " + url);
            }
            if (serverGrayProperty.getOpen()) {
                log.info("当前开启了灰度服务模式");
                return this.choose(exchange).doOnNext((response) -> {
                    if (!response.hasServer()) {
                        throw NotFoundException.create(true, "Unable to find instance for " + url.getHost());
                    } else {
                        URI uri = exchange.getRequest().getURI();
                        log.info("uri:" + uri);
                        String overrideScheme = null;
                        if (schemePrefix != null) {
                            overrideScheme = url.getScheme();
                        }

                        DelegatingServiceInstance serviceInstance = new DelegatingServiceInstance((ServiceInstance) response.getServer(), overrideScheme);
                        URI requestUrl = this.reconstructURI(serviceInstance, uri);
                        if (log.isTraceEnabled()) {
                            log.trace("LoadBalancerClientFilter url chosen: " + requestUrl);
                        }

                        exchange.getAttributes().put(ServerWebExchangeUtils.GATEWAY_REQUEST_URL_ATTR, requestUrl);
                    }
                }).then(chain.filter(exchange));
            } else {
                return super.filter(exchange, chain);
            }
        } else {
            return chain.filter(exchange);
        }
    }

    private Mono<Response<ServiceInstance>> choose(ServerWebExchange exchange) {
        URI uri = (URI) exchange.getAttribute(ServerWebExchangeUtils.GATEWAY_REQUEST_URL_ATTR);
        GrayRoundRobinLoadBalancer loadBalancer = new GrayRoundRobinLoadBalancer(grayClientFactory.getLazyProvider(uri.getHost(), ServiceInstanceListSupplier.class), uri.getHost(), serverGrayProperty);
        return loadBalancer.choose(this.createRequest(exchange));
    }

    private Request createRequest(ServerWebExchange exchange) {
        HttpHeaders headers = exchange.getRequest().getHeaders();
        return new DefaultRequest<>(headers);
    }

    protected URI reconstructURI(ServiceInstance serviceInstance, URI original) {
        return LoadBalancerUriTools.reconstructURI(serviceInstance, original);
    }
}

2.GrayRoundRobinLoadBalancer

核心方法
1.choose
2.getInstanceResponse

import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.ObjectProvider;
import org.springframework.cloud.client.ServiceInstance;
import org.springframework.cloud.client.loadbalancer.reactive.DefaultResponse;
import org.springframework.cloud.client.loadbalancer.reactive.Request;
import org.springframework.cloud.client.loadbalancer.reactive.Response;
import org.springframework.cloud.loadbalancer.core.NoopServiceInstanceListSupplier;
import org.springframework.cloud.loadbalancer.core.ReactorServiceInstanceLoadBalancer;
import org.springframework.cloud.loadbalancer.core.ServiceInstanceListSupplier;
import org.springframework.http.HttpHeaders;
import reactor.core.publisher.Mono;

import java.util.List;
import java.util.Random;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;

/**
 * @author admin
 */
@Slf4j
public class GrayRoundRobinLoadBalancer implements ReactorServiceInstanceLoadBalancer {

    private final ObjectProvider<ServiceInstanceListSupplier> serviceInstanceListSupplierProvider;
    private final String serviceId;

    private final AtomicInteger position = new AtomicInteger(new Random().nextInt(1000));

    public GrayRoundRobinLoadBalancer(ObjectProvider<ServiceInstanceListSupplier> serviceInstanceListSupplierProvider, String serviceId, ServerGrayProperty serverGrayProperty) {
        this.serviceId = serviceId;
        this.serviceInstanceListSupplierProvider = serviceInstanceListSupplierProvider;
    }

    @Override
    public Mono<Response<ServiceInstance>> choose(Request request) {
        HttpHeaders headers = (HttpHeaders) request.getContext();
        ServiceInstanceListSupplier supplier = serviceInstanceListSupplierProvider.getIfAvailable(NoopServiceInstanceListSupplier::new);
        return supplier.get().next().map(list -> getInstanceResponse(list, headers));
    }

    /**
     * 选取服务实例
     *
     * @param instances
     * @param headers
     * @return
     */
    private Response<ServiceInstance> getInstanceResponse(List<ServiceInstance> instances, HttpHeaders headers) {
        List<ServiceInstance> serviceInstances;
        serviceInstances = instances.stream()
                .filter(instance -> {
                    //根据请求头中的版本号信息，选取注册中心中的相应服务实例
                    String version = headers.getFirst("Version");
                    if (version != null) {
                        return version.equals(instance.getMetadata().get("version"));
                    } else {
                        return true;
                    }
                }).collect(Collectors.toList());
        if (serviceInstances.isEmpty()) {
            if (log.isWarnEnabled()) {
                log.warn("gray No servers available for service: " + serviceId);
            }
//            return new EmptyResponse();
            log.warn("服务：" + serviceId + "未找到灰度节点，使用正常的节点");
            serviceInstances = instances;
        }
        int pos = Math.abs(this.position.incrementAndGet());
        ServiceInstance instance = serviceInstances.get(pos % serviceInstances.size());
        log.info("使用的服务是:" + instance.getServiceId());
        log.info("使用的节点url是:" + instance.getHost() + ":" + instance.getPort());
        return new DefaultResponse(instance);
    }
}

3.ServerGrayProperty

package t.gray;


import com.alibaba.nacos.common.utils.MapUtils;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;

import java.util.Map;

/**
 * @author xx
 * @desc 灰度发布服务配置
 */
@Data
@Component
@ConfigurationProperties(prefix = "server.gray")
public class ServerGrayProperty {

    private Boolean open;

    private Integer maxTryTimes;

    private Map<String, Config> config;

    public Config getConfig(String name) {
        if (MapUtils.isEmpty(config)) {
            return null;
        }
        return config.get(name);
    }

    @Data
    public static class Config {
        private String version;
        private Double rate;
    }

    public Integer getMaxTryTimes() {
        return maxTryTimes == null || maxTryTimes <= 0 ? 10 : maxTryTimes;
    }

    public Boolean getOpen() {
        return this.open;
    }
}

4.gateway.yml

# 服务灰度发布配置
server:
  gray:
  # 是否开启灰度 true  false
    open: true
    config:
      # 需要管理发布的服务server-id，当前为 producer 服务
      server-consumer:
        # 需要灰度发布的版本
        version: 2.0
      server-provider:
        # 需要灰度发布的版本
        version: 2.0
  port: 10000
#开启端点
# 暴露监控断点

management:
  endpoints:
    web:
      exposure:
        include:
          - '*'
      health:
        show-details: always
spring:
  application:
    name: gateway
    profiles:
      active: local
  cloud:
    loadbalancer:
      ribbon:
        enabled: false
    gateway:
      discovery:
        locator:
          #开启服务注册和发现功能
          enabled: true
          #将服务名称转换为小写
          lower-case-service-id: true
      routes:
        - id: server-provider
          uri: lb://server-provider
          predicates:
            - Path=/server-provider/**
          filters:
            #将路由第一级去掉
            # - StripPrefix=1
            - name: Hystrix
              args:
                name: fallback
                fallbackUri: forward:/fallback
            - name: RequestRateLimiter
              args:
                # 使用SpEL名称引用Bean，与上面新建的RateLimiterConfig类中的bean的name相同
                key-resolver: '#{@MyKeyResolver}'
                # 每秒最大访问次数
                redis-rate-limiter.replenishRate: 2
                # 令牌桶最大容量
                redis-rate-limiter.burstCapacity: 10
        - id: server-consumer
          uri: lb://server-consumer
          predicates:
            - Path=/server-consumer/**
          filters:
            #将路由第一级去掉
            - StripPrefix=1
            - name: Hystrix
              args:
                name: fallback
                fallbackUri: forward:/fallback
            - name: RequestRateLimiter
              args:
                # 使用SpEL名称引用Bean，与上面新建的RateLimiterConfig类中的bean的name相同
                key-resolver: '#{@MyKeyResolver}'
                # 每秒最大访问次数
                redis-rate-limiter.replenishRate: 2
                # 令牌桶最大容量
                redis-rate-limiter.burstCapacity: 10

  redis:
    host: 1.14.92.8
    port: 6330
    database: 8
    password: 2100redis@@



# feign配置
feign:
  hystrix:
    enabled: true
  httpclient:
    # Feign使用Apache的http client
    enabled: true
  client:
    config:
      default:
        decode404: true

# 断路器的超时时间需要大于ribbon的超时时间，不然不会触发重试。
hystrix:
  command:
    default:
      execution:
        isolation:
          strategy: SEMAPHORE
          thread:
            timeoutInMilliseconds: 3000
          #strategy: THREAD

ribbon:
  ReadTimeout: 2000
  ConnectTimeout: 2000
  #对所有操作请求都进行重试
  OkToRetryOnAllOperations: true
  #切换实例的重试次数
  MaxAutoRetriesNextServer: 1
  #当前实例的重试次数
  maxAutoRetries: 1

2.服务端灰度逻辑（feign内部调用灰度逻辑）

1.FeignRequestInterceptor

package t.config;

import com.alibaba.csp.sentinel.util.StringUtil;
import com.st.common.GrayscaleThreadLocalEnvironment;
import com.st.consts.TraceLogConstants;
import feign.RequestInterceptor;
import feign.RequestTemplate;
import feign.Retryer;
import lombok.extern.slf4j.Slf4j;
import org.slf4j.MDC;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.web.context.request.RequestContextHolder;
import org.springframework.web.context.request.ServletRequestAttributes;

import javax.servlet.http.HttpServletRequest;
import java.util.Enumeration;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Objects;

/**
 * @author x
 */
@Slf4j
@Configuration
public class FeignRequestInterceptor implements RequestInterceptor {

    /**
     * 默认关闭feign的重试功能，feign默认会进行五次重试
     */
    @Bean
    public Retryer feignRetryer() {
        return Retryer.NEVER_RETRY;
    }

    @Override
    public void apply(RequestTemplate template) {
        HttpServletRequest httpServletRequest = getHttpServletRequest();
        Map<String, String> headers = getHeaders(httpServletRequest);
        String traceId = MDC.get(TraceLogConstants.TRACE_ID);
        if(StringUtil.isNotEmpty(traceId)){
            template.header(TraceLogConstants.TRACE_ID, traceId);
        }
        log.info("--consumer服务---FeignRequestInterceptor - ： " + headers.toString());
        for (Map.Entry<String, String> entry : headers.entrySet()) {
            //② 设置请求头到新的Request中
            template.header(entry.getKey(), entry.getValue());
        }
    }

    //获取请求对象
    private HttpServletRequest getHttpServletRequest() {
        try {
            return ((ServletRequestAttributes) Objects.requireNonNull(RequestContextHolder.getRequestAttributes())).getRequest();
        } catch (Exception e) {
            log.info("REQ1001" + "请求信息不能为空");
            return null;
        }
    }

    /**
     * 获取原请求头
     */
    private Map<String, String> getHeaders(HttpServletRequest request) {
        Map<String, String> map = new LinkedHashMap<>();
        Enumeration<String> enumeration = request.getHeaderNames();
        if (enumeration != null) {
            while (enumeration.hasMoreElements()) {
                String key = enumeration.nextElement();
                String value = request.getHeader(key);
                //将灰度标记的请求头透传给下个服务
                if (key.equals("version") && "2.0".equals(value)) {
                    //① 保存灰度发布的标记
                    GrayscaleThreadLocalEnvironment.set("2.0");
                    map.put(key, value);
                }

            }
        }
        return map;
    }
}

2.GrayRule

核心方法choose()

import cn.hutool.core.util.ObjectUtil;
import com.alibaba.cloud.nacos.ribbon.NacosServer;
import com.google.common.base.Optional;
import com.netflix.client.config.IClientConfig;
import com.netflix.loadbalancer.Server;
import com.netflix.loadbalancer.ZoneAvoidanceRule;
import lombok.extern.slf4j.Slf4j;
import org.springframework.util.StringUtils;

import java.util.ArrayList;
import java.util.List;

/**
 * 灰度规则rule
 * feign内部调用，处理选择服务实例
 *
 * @author x
 */
@Slf4j
public class GrayRule extends ZoneAvoidanceRule {
    @Override
    public void initWithNiwsConfig(IClientConfig clientConfig) {
    }

    @Override
    public Server choose(Object key) {
        log.error("-----------------灰度Ribbon---------GrayRule-------------------");
        try {
            //从ThreadLocal中获取灰度标记
            String version = GrayscaleThreadLocalEnvironment.get();
            log.info("网关服务-从ThreadLocal中获取灰度标记: " + version);
            if (ObjectUtil.isEmpty(version)) {
                return super.choose(key);
            } else {
                //获取所有服务
                List<Server> serverList = this.getLoadBalancer().getAllServers();
                log.info("网关服务所有服务总数:" + serverList.size() + ":" + serverList.toString());
                //灰度发布的服务
                List<Server> grayServerList = new ArrayList<>();
                //正常的服务
                List<Server> normalServerList = new ArrayList<>();
                for (Server server : serverList) {
                    NacosServer nacosServer = (NacosServer) server;
                    //从nacos中获取元素剧进行匹配
                    if (nacosServer.getMetadata().containsKey("version")
                            && nacosServer.getMetadata().get("version").equals("2.0")) {
                        grayServerList.add(server);
                    } else {
                        normalServerList.add(server);
                    }
                }
                log.info("网关服务---灰度发布的服务grayServerList----数量:{},{}", grayServerList.size(), grayServerList.toString());
                log.info("网关服务---正常的服务normalServerList-----数量:{},{}", normalServerList.size(), normalServerList.toString());
                if (null == version) {
                    return originChoose(serverList, key, "全部服务");
                }
                //如果被标记为灰度发布，则调用灰度发布的服务
                if ("2.0".equals(version)) {
                    Server grayServer = originChoose(grayServerList, key, "灰度服务");
                    if (null == grayServer || StringUtils.isEmpty(grayServer)) {
                        log.info("无灰度服务或灰度服务列表中没有可用的服务,为保证服务能够正常进行，则将正式环境服务返回");
                        grayServer = originChoose(normalServerList, key, "正常服务");
                    }
                    return grayServer;
                } else {
                    return originChoose(normalServerList, key, "正常服务");
                }
            }
        } finally {
            //清除灰度标记
            GrayscaleThreadLocalEnvironment.remove();
        }
    }

    private Server originChoose(List<Server> noMetaServerList, Object key, String serverName) {
        Optional<Server> server = getPredicate().chooseRoundRobinAfterFiltering(noMetaServerList, key);
        if (server.isPresent()) {
            Server server1 = server.get();
            log.info("网关服务--最终选择的是：【" + serverName + "】服务列表: " + noMetaServerList.toString());
            log.info("网关服务--最终选择的服务是【" + serverName + "】: " + server1);
            return server1;
        } else {
            return null;
        }
    }
}

3.GrayRuleConfig

import org.springframework.context.annotation.Bean;

/**
 * @author
 * 灰度部署的负载规则配置类
 * 注意：这个类一定不要被Spring Boot 扫描进入IOC容器中，一旦扫描进入则对全部的服务都将生效
 */
public class GrayRuleConfig {
    @Bean
    public GrayRule grayRule(){
        return new GrayRule();
    }
}

4.GrayscaleThreadLocalEnvironment

public class GrayscaleThreadLocalEnvironment {

    private static ThreadLocal<String> environment = new ThreadLocal();


    public static String get() {
        return environment.get();
    }

    public static void set(String value) {
        environment.set(value);
    }
    public static void remove() {
        environment.remove();
    }
}

5.服务端灰度标记

metadata

spring:
  application:
    name: server-provider
  profiles:
    active: local
  cloud:
    nacos:
      config:
        username: ${nacos.username}
        password: ${nacos.password}
        server-addr: 127.0.0.1:8848
        file-extension: yml
        namespace: ${spring.profiles.active}
        shared-configs:
          - dataId: common.yml
            refresh: true
          - dataId: server-provider.yml
            refresh: true
      discovery:
        namespace: ${spring.profiles.active}
        username: ${nacos.username}
        password: ${nacos.password}
        server-addr: 127.0.0.1:8848
        metadata:
          # 1.0为正式版本  非1.0都认为是灰度版本
          version: 2.0