Dubbo的负载均衡算法源码分析

Dubbo提供了四种负载均衡：RandomLoadBalance，RoundRobinLoadBalance，LeastActiveLoadBalance，ConsistentHashLoadBalance。

这里顺便说下Dubbo的负载均衡是针对单个客户端的，不是全局的。
以下代码基于2.7.2-SNAPSHOT版本。

LoadBalance

LoadBalance接口只提供了一个对外暴露的方法:
<T> Invoker<T> select(List<Invoker<T>> invokers, URL url, Invocation invocation) throws RpcException;

AbstractLoadBalance

AbstractLoadBalance使用模板设计模式，具体负载均衡算法由子类的doSelect实现

public abstract class AbstractLoadBalance implements LoadBalance {
    
    //预热权重计算，provider刚启动权重在预热时间内随启动时间逐渐增加，最小为1
    static int calculateWarmupWeight(int uptime, int warmup, int weight) {
        int ww = (int) ((float) uptime / ((float) warmup / (float) weight));
        return ww < 1 ? 1 : (ww > weight ? weight : ww);
    }

    //模板方法，参数判断
    public <T> Invoker<T> select(List<Invoker<T>> invokers, URL url, Invocation invocation) {
        if (CollectionUtils.isEmpty(invokers)) {
            return null;
        }
        if (invokers.size() == 1) {
            return invokers.get(0);
        }
        return doSelect(invokers, url, invocation);
    }
    
    //真正执行负载均衡的方法
    protected abstract <T> Invoker<T> doSelect(List<Invoker<T>> invokers, URL url, Invocation invocation);


    //计算权重
    protected int getWeight(Invoker<?> invoker, Invocation invocation) {
        int weight = invoker.getUrl().getMethodParameter(invocation.getMethodName(), Constants.WEIGHT_KEY, Constants.DEFAULT_WEIGHT);
        if (weight > 0) {
            long timestamp = invoker.getUrl().getParameter(Constants.REMOTE_TIMESTAMP_KEY, 0L);
            if (timestamp > 0L) {
                int uptime = (int) (System.currentTimeMillis() - timestamp);
                int warmup = invoker.getUrl().getParameter(Constants.WARMUP_KEY, Constants.DEFAULT_WARMUP);//预热时间，默认为10分钟
                if (uptime > 0 && uptime < warmup) {//预热
                    weight = calculateWarmupWeight(uptime, warmup, weight);
                }
            }
        }
        return weight >= 0 ? weight : 0;
    }
}

RandomLoadBalance

public class RandomLoadBalance extends AbstractLoadBalance {

    public static final String NAME = "random";

    @Override
    protected <T> Invoker<T> doSelect(List<Invoker<T>> invokers, URL url, Invocation invocation) {
        int length = invokers.size();//invoker个数
        boolean sameWeight = true;//每个invoker都有相同权重
        int[] weights = new int[length];//权重数组
        int firstWeight = getWeight(invokers.get(0), invocation);//第一个权重
        weights[0] = firstWeight;
        int totalWeight = firstWeight;//总权重
        
        //计算总权重和判断权重是否相同
        for (int i = 1; i < length; i++) {
            int weight = getWeight(invokers.get(i), invocation);
            weights[i] = weight;
            totalWeight += weight;
            if (sameWeight && weight != firstWeight) {
                sameWeight = false;
            }
        }
        
        //权重不相同
        if (totalWeight > 0 && !sameWeight) {
            //得到一个在[0,totalWeight)的偏移量，然后这个偏移量所在的invoker
            int offset = ThreadLocalRandom.current().nextInt(totalWeight);
            for (int i = 0; i < length; i++) {
                offset -= weights[i];
                if (offset < 0) {
                    return invokers.get(i);
                }
            }
        }
        
        //权重相同，直接随机[0,length)
        return invokers.get(ThreadLocalRandom.current().nextInt(length));
    }
}

RoundRobinLoadBalance

public class RoundRobinLoadBalance extends AbstractLoadBalance {
    public static final String NAME = "roundrobin";
    
    //循环周期，如果在这个周期内invoker没有被客户端获取，那么该invoker对应的轮询记录将被删除。
    private static final int RECYCLE_PERIOD = 60000;
    
    //自己理解为轮询记录，一个invoker对应一个WeightedRoundRobin
    protected static class WeightedRoundRobin {
        private int weight;//权重
        //目前权重，随着轮询次数改变，每个轮询周期增加weight，如果invoker被调用减去总权重
        private AtomicLong current = new AtomicLong(0);
        //最后一次获取到invoker的时间，判断invoker是否失去连接
        private long lastUpdate;
        public int getWeight() {
            return weight;
        }
        public void setWeight(int weight) {
            this.weight = weight;
            current.set(0);
        }
        public long increaseCurrent() {
            return current.addAndGet(weight);
        }
        public void sel(int total) {
            current.addAndGet(-1 * total);
        }
        public long getLastUpdate() {
            return lastUpdate;
        }
        public void setLastUpdate(long lastUpdate) {
            this.lastUpdate = lastUpdate;
        }
    }
    
    //保存轮询记录，方法的全限定路径--->Map(invoker的IdentityString---->invoker的WeightedRoundRobin)
    private ConcurrentMap<String, ConcurrentMap<String, WeightedRoundRobin>> methodWeightMap = new ConcurrentHashMap<String, ConcurrentMap<String, WeightedRoundRobin>>();
    //修改methodWeightMap的锁
    private AtomicBoolean updateLock = new AtomicBoolean();
    
    //从methodWeightMap获取所有invoker对应WeightedRoundRobin
    protected <T> Collection<String> getInvokerAddrList(List<Invoker<T>> invokers, Invocation invocation) {
        String key = invokers.get(0).getUrl().getServiceKey() + "." + invocation.getMethodName();
        Map<String, WeightedRoundRobin> map = methodWeightMap.get(key);
        if (map != null) {
            return map.keySet();
        }
        return null;
    }
    
    @Override
    protected <T> Invoker<T> doSelect(List<Invoker<T>> invokers, URL url, Invocation invocation) {
        //从methodWeightMap获取所有invoker对应WeightedRoundRobin
        String key = invokers.get(0).getUrl().getServiceKey() + "." + invocation.getMethodName();
        ConcurrentMap<String, WeightedRoundRobin> map = methodWeightMap.get(key);
        if (map == null) {
            methodWeightMap.putIfAbsent(key, new ConcurrentHashMap<String, WeightedRoundRobin>());
            map = methodWeightMap.get(key);
        }


        int totalWeight = 0;//总权重
        long maxCurrent = Long.MIN_VALUE;//最大目前权重
        long now = System.currentTimeMillis();
        Invoker<T> selectedInvoker = null;//选中invoker
        WeightedRoundRobin selectedWRR = null;//选中invoker对应的WeightedRoundRobin


        for (Invoker<T> invoker : invokers) {
            //获取invoker对应的WeightedRoundRobin
            String identifyString = invoker.getUrl().toIdentityString();
            WeightedRoundRobin weightedRoundRobin = map.get(identifyString);
            int weight = getWeight(invoker, invocation);
            
            if (weightedRoundRobin == null) {
                weightedRoundRobin = new WeightedRoundRobin();
                weightedRoundRobin.setWeight(weight);
                map.putIfAbsent(identifyString, weightedRoundRobin);
            }

            //更新weight
            if (weight != weightedRoundRobin.getWeight()) {
                //weight changed
                weightedRoundRobin.setWeight(weight);
            }
            
            //一次轮询周期增加一次权重
            long cur = weightedRoundRobin.increaseCurrent();
            //更新invoker获取时间
            weightedRoundRobin.setLastUpdate(now);
            //选中最大的目前权重
            if (cur > maxCurrent) {
                maxCurrent = cur;
                selectedInvoker = invoker;
                selectedWRR = weightedRoundRobin;
            }
            totalWeight += weight;
        }
        
        //如果invokers发生改变（增加或者失去连接），更新map
        if (!updateLock.get() && invokers.size() != map.size()) {
            if (updateLock.compareAndSet(false, true)) {
                try {
                    // copy -> modify -> update reference
                    ConcurrentMap<String, WeightedRoundRobin> newMap = new ConcurrentHashMap<String, WeightedRoundRobin>();
                    newMap.putAll(map);
                    Iterator<Entry<String, WeightedRoundRobin>> it = newMap.entrySet().iterator();
                    while (it.hasNext()) {
                        //如果在60000ms内invoker没有被客户端获取，则认为该invoker下线，那么该invoker对应的轮询记录将被删除。
                        Entry<String, WeightedRoundRobin> item = it.next();
                        if (now - item.getValue().getLastUpdate() > RECYCLE_PERIOD) {
                            it.remove();
                        }
                    }
                    methodWeightMap.put(key, newMap);
                } finally {
                    updateLock.set(false);
                }
            }
        }
        
        //选中的invoker减去总权重
        if (selectedInvoker != null) {
            selectedWRR.sel(totalWeight);
            return selectedInvoker;
        }
        // should not happen here
        return invokers.get(0);
    }

}

轮询负载均衡存在一个问题，不推荐使用，文档说法如下：

存在慢的提供者累积请求的问题，比如：第二台机器很慢，但没挂，当请求调到第二台时就卡在那，久而久之，所有请求都卡在调到第二台上。

LeastActiveLoadBalance

public class LeastActiveLoadBalance extends AbstractLoadBalance {

    public static final String NAME = "leastactive";

    @Override
    protected <T> Invoker<T> doSelect(List<Invoker<T>> invokers, URL url, Invocation invocation) {
        // invokers个数
        int length = invokers.size();
        // 最小活跃度
        int leastActive = -1;
        // 具有最小活跃度的invoker个数
        int leastCount = 0;
        // 每个具有最小活跃度的invoker的下标
        int[] leastIndexes = new int[length];
        // 权重数组
        int[] weights = new int[length];
        // 所有最小活跃度invoker的总权重
        int totalWeight = 0;
        // 第一个最小活跃度invoker的权重
        int firstWeight = 0;
        // 判断每个最小活跃度invoker的权重是否相同
        boolean sameWeight = true;


        for (int i = 0; i < length; i++) {
            Invoker<T> invoker = invokers.get(i);
            // 获取活跃度
            int active = RpcStatus.getStatus(invoker.getUrl(), invocation.getMethodName()).getActive();
            // 获取权重
            int afterWarmup = getWeight(invoker, invocation);
            weights[i] = afterWarmup;

            // 记录最小活跃度
            if (leastActive == -1 || active < leastActive) {
                // Reset the active number of the current invoker to the least active number
                leastActive = active;
                // Reset the number of least active invokers
                leastCount = 1;
                // Put the first least active invoker first in leastIndexs
                leastIndexes[0] = i;
                // Reset totalWeight
                totalWeight = afterWarmup;
                // Record the weight the first least active invoker
                firstWeight = afterWarmup;
                // Each invoke has the same weight (only one invoker here)
                sameWeight = true;
                // If current invoker's active value equals with leaseActive, then accumulating.
            } else if (active == leastActive) {//有多个最小活跃度
                // Record the index of the least active invoker in leastIndexs order
                leastIndexes[leastCount++] = i;
                // Accumulate the total weight of the least active invoker
                totalWeight += afterWarmup;
                // If every invoker has the same weight?
                if (sameWeight && i > 0
                        && afterWarmup != firstWeight) {
                    sameWeight = false;
                }
            }
        }

        // 最小活跃度的只有一个直接返回
        if (leastCount == 1) {
            return invokers.get(leastIndexes[0]);
        }
        
        //多个最小活跃度，处理跟随机负载均衡一样
        if (!sameWeight && totalWeight > 0) {
            int offsetWeight = ThreadLocalRandom.current().nextInt(totalWeight);
            for (int i = 0; i < leastCount; i++) {
                int leastIndex = leastIndexes[i];
                offsetWeight -= weights[leastIndex];
                if (offsetWeight < 0) {
                    return invokers.get(leastIndex);
                }
            }
        }
        return invokers.get(leastIndexes[ThreadLocalRandom.current().nextInt(leastCount)]);
    }
}

LeastActiveLoadBalance与ActiveLimitFilter一起使用，ActiveLimitFilter负责记录每个invoker调用的活跃度。

ConsistentHashLoadBalance

一致性hash算法

一致性hash将整个hash空间组织成一个环，范围在0-2^32-1

然后将各个节点使用唯一标志（ip或者hostname等）hash分布在环上

最后将数据key使用相同hash算法定位在环上位置，顺时针遇到的第一台服务器就是该数据的存储节点

算法的容错性

此外如果某个节点宕机或增加节点，影响的仅仅是邻近的下一个节点

虚拟节点

当节点个数太少时，容易出现节点分布不均匀导致数据倾斜问题

为了解决数据倾斜问题，引入虚拟节点机制，将节点多次hash，生成多个节点位置，具体做法可以在节点标志后面加编号区分。

Dubbo实现

public class ConsistentHashLoadBalance extends AbstractLoadBalance {
    public static final String NAME = "consistenthash";

    private final ConcurrentMap<String, ConsistentHashSelector<?>> selectors = new ConcurrentHashMap<String, ConsistentHashSelector<?>>();

    @SuppressWarnings("unchecked")
    @Override
    protected <T> Invoker<T> doSelect(List<Invoker<T>> invokers, URL url, Invocation invocation) {
        String methodName = RpcUtils.getMethodName(invocation);
        String key = invokers.get(0).getUrl().getServiceKey() + "." + methodName;
        int identityHashCode = System.identityHashCode(invokers);
        ConsistentHashSelector<T> selector = (ConsistentHashSelector<T>) selectors.get(key);
        if (selector == null || selector.identityHashCode != identityHashCode) {
            selectors.put(key, new ConsistentHashSelector<T>(invokers, methodName, identityHashCode));
            selector = (ConsistentHashSelector<T>) selectors.get(key);
        }
        return selector.select(invocation);
    }

    private static final class ConsistentHashSelector<T> {

        private final TreeMap<Long, Invoker<T>> virtualInvokers;

        private final int replicaNumber;

        private final int identityHashCode;

        private final int[] argumentIndex;

        ConsistentHashSelector(List<Invoker<T>> invokers, String methodName, int identityHashCode) {
            this.virtualInvokers = new TreeMap<Long, Invoker<T>>();
            this.identityHashCode = identityHashCode;
            URL url = invokers.get(0).getUrl();
            //虚拟节点个数
            this.replicaNumber = url.getMethodParameter(methodName, HASH_NODES, 160);
            //hash的参数下标
            String[] index = Constants.COMMA_SPLIT_PATTERN.split(url.getMethodParameter(methodName, HASH_ARGUMENTS, "0"));
            argumentIndex = new int[index.length];
            for (int i = 0; i < index.length; i++) {
                argumentIndex[i] = Integer.parseInt(index[i]);
            }
            
            //创建虚拟节点
            for (Invoker<T> invoker : invokers) {
                String address = invoker.getUrl().getAddress();
                for (int i = 0; i < replicaNumber / 4; i++) {
                    byte[] digest = md5(address + i);
                    for (int h = 0; h < 4; h++) {
                        long m = hash(digest, h);
                        virtualInvokers.put(m, invoker);
                    }
                }
            }
        }

        public Invoker<T> select(Invocation invocation) {
            String key = toKey(invocation.getArguments());
            byte[] digest = md5(key);
            return selectForKey(hash(digest, 0));
        }

        private String toKey(Object[] args) {
            StringBuilder buf = new StringBuilder();
            for (int i : argumentIndex) {
                if (i >= 0 && i < args.length) {
                    buf.append(args[i]);
                }
            }
            return buf.toString();
        }

        private Invoker<T> selectForKey(long hash) {
            Map.Entry<Long, Invoker<T>> entry = virtualInvokers.ceilingEntry(hash);
            if (entry == null) {
                entry = virtualInvokers.firstEntry();
            }
            return entry.getValue();
        }

        private long hash(byte[] digest, int number) {
            return (((long) (digest[3 + number * 4] & 0xFF) << 24)
                    | ((long) (digest[2 + number * 4] & 0xFF) << 16)
                    | ((long) (digest[1 + number * 4] & 0xFF) << 8)
                    | (digest[number * 4] & 0xFF))
                    & 0xFFFFFFFFL;
        }

        private byte[] md5(String value) {
            MessageDigest md5;
            try {
                md5 = MessageDigest.getInstance("MD5");
            } catch (NoSuchAlgorithmException e) {
                throw new IllegalStateException(e.getMessage(), e);
            }
            md5.reset();
            byte[] bytes = value.getBytes(StandardCharsets.UTF_8);
            md5.update(bytes);
            return md5.digest();
        }

    }

}

参考资料

http://huangzehong.me/2018/09/06/20180906-%E3%80%90Dubbo%E6%BA%90%E7%A0%81%E5%88%86%E6%9E%90%E3%80%91%E5%9B%9B%E7%A7%8D%E8%B4%9F%E8%BD%BD%E5%9D%87%E8%A1%A1/
https://www.cnblogs.com/lpfuture/p/5796398.html

转载于:https://www.cnblogs.com/wuweishuo/p/10802509.html