Ribbon的ZoneAwareLoadBalancer

一 ZoneAwareLoadBalancer在类图中的位置

二 引入原因

DynamicServerListLoadBalancer，采用的是线性轮询的方式来选择调用服务实例，该算法实现简单并没有区域Zone的概念，所以它会把所有实例视为一个Zone下的节点来看待，这样就会周期性地产生跨区域访问的情况，由于跨区域会产生更高的延迟，这些实例主要以防止区域故障实现高可用的目的而不能作为常规访问的实例。所以在多区域部署的情况下会有一定的性能问题。ZoneAwareLoadBalancer可用避免这样的问题。

三 ZoneAwareLoadBalancer源码解读

//下面是父类DynamicServerListLoadBalancer的关键代码
    @Override
    public void setServersList(List lsrv) {
        super.setServersList(lsrv);
        List<T> serverList = (List<T>) lsrv;
        Map<String, List<Server>> serversInZones = new HashMap<String, List<Server>>();
        for (Server server : serverList) {
            // make sure ServerStats is created to avoid creating them on hot
            // path
            getLoadBalancerStats().getSingleServerStat(server);
            String zone = server.getZone();
            if (zone != null) {
                zone = zone.toLowerCase();
                List<Server> servers = serversInZones.get(zone);
                if (servers == null) {
                    servers = new ArrayList<Server>();
                    //按区域Zone分组实例列表
                    serversInZones.put(zone, servers);
                }
                servers.add(server);
            }
        }
        setServerListForZones(serversInZones);
    }
//
    //下面三个函数实现的功能是为负载均衡器中的LoadBalancerStats对象创建ZoneStats并放入zoneStatsMap
    //集合中，每个Zone对应一个ZoneStats，它用于存储每个Zone的一些状态和统计信息
    protected void setServerListForZones(
            Map<String, List<Server>> zoneServersMap) {
        LOGGER.debug("Setting server list for zones: {}", zoneServersMap);
        getLoadBalancerStats().updateZoneServerMapping(zoneServersMap);
    }
    public void updateZoneServerMapping(Map<String, List<Server>> map) {
        upServerListZoneMap = new ConcurrentHashMap<String, List<? extends Server>>(map);
        // make sure ZoneStats object exist for available zones for monitoring purpose
        for (String zone: map.keySet()) {
            getZoneStats(zone);
        }
    }
    private ZoneStats getZoneStats(String zone) {
        zone = zone.toLowerCase();
        ZoneStats zs = zoneStatsMap.get(zone);
        if (zs == null){
            zoneStatsMap.put(zone, new ZoneStats(this.getName(), zone, this));
            zs = zoneStatsMap.get(zone);
        }
        return zs;
    }
//

public class ZoneAwareLoadBalancer<T extends Server> extends DynamicServerListLoadBalancer<T> {

    private ConcurrentHashMap<String, BaseLoadBalancer> balancers = new ConcurrentHashMap<String, BaseLoadBalancer>();
    
    private static final Logger logger = LoggerFactory.getLogger(ZoneAwareLoadBalancer.class);
            
    private volatile DynamicDoubleProperty triggeringLoad;

    private volatile DynamicDoubleProperty triggeringBlackoutPercentage;

    private static final DynamicBooleanProperty ENABLED = DynamicPropertyFactory.getInstance().getBooleanProperty("ZoneAwareNIWSDiscoveryLoadBalancer.enabled", true);
            
    void setUpServerList(List<Server> upServerList) {
        this.upServerList = upServerList;
    }
    
    public ZoneAwareLoadBalancer() {
        super();
    }

    @Deprecated
    public ZoneAwareLoadBalancer(IClientConfig clientConfig, IRule rule,
            IPing ping, ServerList<T> serverList, ServerListFilter<T> filter) {
        super(clientConfig, rule, ping, serverList, filter);
    }

    public ZoneAwareLoadBalancer(IClientConfig clientConfig, IRule rule,
                                 IPing ping, ServerList<T> serverList, ServerListFilter<T> filter,
                                 ServerListUpdater serverListUpdater) {
        super(clientConfig, rule, ping, serverList, filter, serverListUpdater);
    }

    public ZoneAwareLoadBalancer(IClientConfig niwsClientConfig) {
        super(niwsClientConfig);
    }

    @Override
    protected void setServerListForZones(Map<String, List<Server>> zoneServersMap) {
        super.setServerListForZones(zoneServersMap);
        if (balancers == null) {
            //存储每个Zone区域对应的负载均衡器。
            balancers = new ConcurrentHashMap<String, BaseLoadBalancer>();
        }
        for (Map.Entry<String, List<Server>> entry: zoneServersMap.entrySet()) {
            String zone = entry.getKey().toLowerCase();
            getLoadBalancer(zone).setServersList(entry.getValue());  //设置对应Zone区域的实例清单
        }
        //对Zone区域中的实例清单的检查，看看是否有zone区域下已经没有实例了，是的话
        //就将Balancers中对应Zone区域的实例列表清空，该操作的作用是为了后续选择节点
        //时，防止过时的Zone区域统计信息干扰具体实例的选择算法。
        for (Map.Entry<String, BaseLoadBalancer> existingLBEntry: balancers.entrySet()) {
            if (!zoneServersMap.keySet().contains(existingLBEntry.getKey())) {
                existingLBEntry.getValue().setServersList(Collections.emptyList());
            }
        }
    }    
        
    @Override
    public Server chooseServer(Object key) {
        //只有当负载均衡器中维护的实例所属的Zone区域的个数大于1的时候才会执行选择策略
        //否则还是使用父类的实现
        if (!ENABLED.get() || getLoadBalancerStats().getAvailableZones().size() <= 1) {
            logger.debug("Zone aware logic disabled or there is only one zone");
            return super.chooseServer(key);
        }
        Server server = null;
        try {
            LoadBalancerStats lbStats = getLoadBalancerStats();
            //为当前负载均衡器中的所有Zone区域分别创建快照，保存在zoneSnapshot中，这些快照中的数据用于后续的算法
            Map<String, ZoneSnapshot> zoneSnapshot = ZoneAvoidanceRule.createSnapshot(lbStats);
            logger.debug("Zone snapshots: {}", zoneSnapshot);
            if (triggeringLoad == null) {
                triggeringLoad = DynamicPropertyFactory.getInstance().getDoubleProperty(
                        "ZoneAwareNIWSDiscoveryLoadBalancer." + this.getName() + ".triggeringLoadPerServerThreshold", 0.2d);
            }

            if (triggeringBlackoutPercentage == null) {
                triggeringBlackoutPercentage = DynamicPropertyFactory.getInstance().getDoubleProperty(
                        "ZoneAwareNIWSDiscoveryLoadBalancer." + this.getName() + ".avoidZoneWithBlackoutPercetage", 0.99999d);
            }
            //获得可用Zone区域的集合，getAvailableZones会通过zoneSnapshot实现可用区域挑选
            Set<String> availableZones = ZoneAvoidanceRule.getAvailableZones(zoneSnapshot, triggeringLoad.get(), triggeringBlackoutPercentage.get());
            logger.debug("Available zones: {}", availableZones);
            if (availableZones != null &&  availableZones.size() < zoneSnapshot.keySet().size()) {
                //随机选择一个Zone区域
                String zone = ZoneAvoidanceRule.randomChooseZone(zoneSnapshot, availableZones);
                logger.debug("Zone chosen: {}", zone);
                if (zone != null) {
                    //获得对应区域的负载均衡器
                    BaseLoadBalancer zoneLoadBalancer = getLoadBalancer(zone);
                    //选择具体的服务实例
                    //在chooseServer中将会使用IRule接口的choose函数来选择具体服务实例。在这里，IRule接口的实现会实现ZoneAvoidanceRule来挑选具体的服务实例。
                    server = zoneLoadBalancer.chooseServer(key);
                }
            }
        } catch (Exception e) {
            logger.error("Error choosing server using zone aware logic for load balancer={}", name, e);
        }
        if (server != null) {
            return server;
        } else {
            logger.debug("Zone avoidance logic is not invoked.");
            return super.chooseServer(key);
        }
    }
     
    //为每个区域创建负载均衡器，并设置负载均衡规则
    @VisibleForTesting
    BaseLoadBalancer getLoadBalancer(String zone) {
        zone = zone.toLowerCase();
        BaseLoadBalancer loadBalancer = balancers.get(zone);
        if (loadBalancer == null) {
            // 创建规则
            IRule rule = cloneRule(this.getRule());
            loadBalancer = new BaseLoadBalancer(this.getName() + "_" + zone, rule, this.getLoadBalancerStats());
            BaseLoadBalancer prev = balancers.putIfAbsent(zone, loadBalancer);
            if (prev != null) {
                loadBalancer = prev;
            }
        }
        return loadBalancer;        
    }

    private IRule cloneRule(IRule toClone) {
        IRule rule;
        if (toClone == null) {
            //没有IRule实例，创建AvailabilityFilteringRule实例
            rule = new AvailabilityFilteringRule();
        } else {
            String ruleClass = toClone.getClass().getName();                
            try {
                //有就克隆一个
                rule = (IRule) ClientFactory.instantiateInstanceWithClientConfig(ruleClass, this.getClientConfig());
            } catch (Exception e) {
                throw new RuntimeException("Unexpected exception creating rule for ZoneAwareLoadBalancer", e);
            }
        }
        return rule;
    }
    
       
    @Override
    public void setRule(IRule rule) {
        super.setRule(rule);
        if (balancers != null) {
            for (String zone: balancers.keySet()) {
                balancers.get(zone).setRule(cloneRule(rule));
            }
        }
    }
}
///
//ZoneAwareLoadBalancer会用到该函数
public static Set<String> getAvailableZones(
        Map<String, ZoneSnapshot> snapshot, double triggeringLoad,
        double triggeringBlackoutPercentage) {
    if (snapshot.isEmpty()) {
        return null;
    }
    Set<String> availableZones = new HashSet<String>(snapshot.keySet());
    if (availableZones.size() == 1) {
        return availableZones;
    }
    Set<String> worstZones = new HashSet<String>();
    double maxLoadPerServer = 0;
    boolean limitedZoneAvailability = false;

    for (Map.Entry<String, ZoneSnapshot> zoneEntry : snapshot.entrySet()) {
        String zone = zoneEntry.getKey();
        ZoneSnapshot zoneSnapshot = zoneEntry.getValue();
        int instanceCount = zoneSnapshot.getInstanceCount();
        if (instanceCount == 0) {
            availableZones.remove(zone);     //剔除实例个数为0的Zone区域
            limitedZoneAvailability = true;
        } else {
            double loadPerServer = zoneSnapshot.getLoadPerServer();
            if (((double) zoneSnapshot.getCircuitTrippedCount())
                    / instanceCount >= triggeringBlackoutPercentage
                    || loadPerServer < 0) {
                //实例平均负载小于0的区域剔除
                //实例故障率（断路器端口次数/实例数）大于等于阈值（默认0,99999）的区域剔除。
                availableZones.remove(zone);
                limitedZoneAvailability = true;
            } else {
                //实例平均负载最差的Zone区域找到，这里最差指的是实例平均负载最高的zone区域
                if (Math.abs(loadPerServer - maxLoadPerServer) < 0.000001d) {
                    worstZones.add(zone);
                } else if (loadPerServer > maxLoadPerServer) {
                    maxLoadPerServer = loadPerServer;
                    worstZones.clear();
                    worstZones.add(zone);
                }
            }
        }
    }
    //如有实例的最大平均负载小于阈值（默认20%）
    //如果没有符合剔除要求的区域
    //符合这两个条件就直接返回所有Zone区域为可用区域
    if (maxLoadPerServer < triggeringLoad && !limitedZoneAvailability) {
        // zone override is not needed here
        return availableZones;
    }
    //随机选择一个最差的区域
    String zoneToAvoid = randomChooseZone(snapshot, worstZones);
    if (zoneToAvoid != null) {
        //干掉最差的区域
        availableZones.remove(zoneToAvoid);
    }
    return availableZones;

}