【Hash】loadbalance 一致性hash算法线上应用

最新推荐文章于 2024-05-21 18:13:02 发布
自驱
最新推荐文章于 2024-05-21 18:13:02 发布
阅读量280
点赞数
分类专栏：架构
本文链接：https://blog.csdn.net/qfzhangwei/article/details/113191469
版权
架构专栏收录该内容
161 篇文章 4 订阅
订阅专栏
1 小结：

小心处理链接泄露问题
listener 监听更新要可能早于注入更新（可能是一定）
权重每次重分布都要更新
2 源码：


import com.alibaba.nacos.api.NacosFactory;
import com.alibaba.nacos.api.config.ConfigService;
import com.alibaba.nacos.api.config.annotation.NacosValue;
import com.alibaba.nacos.api.config.listener.Listener;
import com.alibaba.nacos.api.exception.NacosException;
import com.google.common.base.Splitter;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.JedisPoolConfig;

import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.io.StringReader;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.Executor;
import java.util.stream.Collectors;

/**
 * 为tair做的一致性hash算法
 * 客户端负载均衡
 * 注意:默认url 划分了300个虚拟节点，可以在配置中心自己定义具体虚拟节点数
 */
@Slf4j
@Component
public class ConsistanceHashBalance {

    private static final String DEFAULT_BALANCE = "consistenthash";
    private static final String ESS_TAIR_HOST = "ess_tair_host_test";
    /**
     * 默认配置
     */
    private static final String HOST = "r-uf6yxxxx.com";
    /**
     * 默认虚拟节点个数
     */
    private static final int DEFAULT_VIRTUAL_SIZE = 300;
    /**
     * 初始化连接超时时间
     */
    private static final int DEFAULT_CONNECTION_TIMEOUT = 50000;
    /**
     * 查询超时时间
     */
    private static final int DEFAULT_SO_TIMEOUT = 20000;

    private static final int PORT = 6379;
    private static final String PASSWORD = "1345678";

    /**
     * 扩容机器Host
      */
    @NacosValue(value = "${ess_tair_host_test}",autoRefreshed = true)
    private List<String> essTairHost;

    @Value("${nacos.config.server-addr}")
    private String serverAddr;

    @Autowired
    @Qualifier("asyncTairSink")
    private Executor asyncTair;

    private volatile String LAST_ESS_TAIR_HOST = "";
    private static final Map<String, JedisHodler> POOLS_CACHE = Maps.newLinkedHashMap();
    private final ConcurrentMap<String, ConsistentHashSelector> selectors = new ConcurrentHashMap<>();

    private JedisPool jedisPool;

    @PostConstruct
    public void init() throws NacosException {
        tryRefresh(POOLS_CACHE, essTairHost);
        addListener();
    }

    @PreDestroy
    public void destroy(){
        for (JedisHodler hodler : POOLS_CACHE.values()) {
            hodler.jedisPool.destroy();
        }
    }

    private void addListener() throws NacosException {
        Properties properties = new Properties();
        properties.put("serverAddr", serverAddr);
        ConfigService configService = NacosFactory.createConfigService(properties);
        String content = configService.getConfig("rec-read", "DEFAULT_GROUP", 5000);
        log.info("【tair 负载均衡】初始化 .... content = {}",content);
        configService.addListener("rec-read", "DEFAULT_GROUP", new Listener() {
            @Override
            public Executor getExecutor() {
                return asyncTair;
            }
            @Override
            public void receiveConfigInfo(String configInfo) {
                log.info("【tair 负载均衡】监听 变更 .... configInfo = {}",configInfo);
                try {
                    Properties receiveProperties = new Properties();
                    receiveProperties.load(new StringReader(configInfo));
                    String property = receiveProperties.getProperty(ESS_TAIR_HOST);
                    log.warn("【tair 负载均衡】监听 old host = {} ",LAST_ESS_TAIR_HOST);
                    log.warn("【tair 负载均衡】监听 new host = {} ",property);
                    if(Objects.equals(LAST_ESS_TAIR_HOST, property)) {
                        log.warn("【tair 负载均衡】监听 ess host 没有发生变更，确认是否符合预期 ");
                        return;
                    }
                    log.info("【tair 负载均衡】监听 接收到变更,执行新的负载均衡 ess_tair_host = {}", property);
                    Map<String, JedisHodler> map = Maps.newLinkedHashMap();
                    // 刷新替换最新的有效连接, 注意不能使用this对象中注入的值，this.essTairHost update after this listener call
                    tryRefresh(map, Splitter.on(",").trimResults().splitToList(property));
                    // 关闭无效缓存链接，有一定的overhead
                    for (Map.Entry<String, JedisHodler> entry : POOLS_CACHE.entrySet()) {
                        if(!map.containsKey(entry.getKey())){
                            log.warn("【tair 负载均衡】监听 关闭无效链接 host = {}", entry.getValue().host);
                            entry.getValue().jedisPool.destroy();
                        }
                    }
                    // 清空
                    POOLS_CACHE.clear();
                    // 更新
                    POOLS_CACHE.putAll(map);
                    // 保存扩展属性
                    LAST_ESS_TAIR_HOST = property;
                }catch (Exception e){
                    log.error("【tair 负载均衡】监听 配置编程加载异常 ",e);
                }
            }
        });
    }

    private void tryRefresh(Map<String, JedisHodler> hodlerMap, List<String> essTairHost) {
        // 参数设置最佳实践可参考：https://help.aliyun.com/document_detail/98726.html
        JedisPoolConfig config = new JedisPoolConfig();

        config.setMaxTotal(32);
        config.setMaxIdle(32);
        config.setMaxIdle(20);

        // 可以覆盖默认集群权重,权重reset
        // others init host1:权重1,host2:权重2 格式
        if(CollectionUtils.isNotEmpty(essTairHost)){
            for (String essHost : essTairHost) {
                String[] split = essHost.split(":");
                String essHostUrl = split[0];
                JedisHodler hodler = POOLS_CACHE.get(essHostUrl);

                if(null == hodler || hodler.jedisPool.isClosed()) {
                    JedisPool essJedisPool = new JedisPool(config, essHostUrl, PORT, DEFAULT_CONNECTION_TIMEOUT,
                            DEFAULT_SO_TIMEOUT, PASSWORD, 0, null);
                    hodlerMap.put(essHostUrl, new JedisHodler(essJedisPool, essHostUrl, Integer.valueOf(split[1])));
                }else{
                    // 原始连接不变，只重置权重
                    hodlerMap.put(essHostUrl, new JedisHodler(hodler.getJedisPool(), essHostUrl, Integer.valueOf(split[1])));
                }
            }
        }

        // 如果自定义，则忽略默认配置
        JedisHodler customDefaultHolder = hodlerMap.get(HOST);
        if(null == customDefaultHolder) {
            JedisHodler deafultHodler = POOLS_CACHE.get(HOST);
            if (null == deafultHodler || deafultHodler.jedisPool.isClosed()) {
                jedisPool = new JedisPool(config, HOST, PORT, DEFAULT_CONNECTION_TIMEOUT,
                        DEFAULT_SO_TIMEOUT, PASSWORD, 0, null);
                hodlerMap.put(HOST, new JedisHodler(jedisPool, HOST, DEFAULT_VIRTUAL_SIZE));
            } else {
                hodlerMap.put(HOST, deafultHodler);
            }
        }

        // 一致性hash
        List<JedisPool> jedisPools = hodlerMap.values().stream().map(JedisHodler::getJedisPool).collect(Collectors.toList());
        int identityHashCode = System.identityHashCode(jedisPools);
        selectors.put(DEFAULT_BALANCE, new ConsistentHashSelector(Lists.newArrayList(hodlerMap.values()), identityHashCode));
        log.info("【tair 负载均衡】refresh identityHashCode = {}, jedisPools = {}", identityHashCode, jedisPools);
    }
    // 暴露调用入口
    public JedisPool doSelect(String key){
        ConsistentHashSelector consistentHashSelector = selectors.get(DEFAULT_BALANCE);
        return consistentHashSelector.select(key);
    }

    @Data
    public static class JedisHodler{
        private JedisPool jedisPool;
        private Integer weight;
        private String host;

        public JedisHodler(JedisPool jedisPool, String host, Integer weight) {
            this.jedisPool = jedisPool;
            this.weight = weight;
            this.host = host;
            log.info("【tair 负载均衡】 节点初始化 host = {} , weight = {}", host, weight);
        }
    }

    private static final class ConsistentHashSelector {
        private final TreeMap<Long, JedisPool> virtualJedisPool;
        // 保留暂时上层做幂等
        private final int identityHashCode;


        /**
         * 参考dubbo一致性hash算法
         *
         */
        ConsistentHashSelector(List<JedisHodler> jedisHolders, int identityHashCode) {
            this.virtualJedisPool = new TreeMap<>();
            this.identityHashCode = identityHashCode;
            // 每个pool一个权重, 此处一致性hash确保，机器地址不变，那么获取环形位置集合也是确定的，
            // 当然扩容缩容时重合部分也是固定不变化的
            for (JedisHodler hodler : jedisHolders) {
                String address = hodler.getHost();
                int virtualCount = 0;
                for (int i = 0; i < hodler.getWeight() / 4; i++) {
                    byte[] digest = md5(address + i);
                    for (int h = 0; h < 4; h++) {
                        long m = hash(digest, h);
                        virtualJedisPool.put(m, hodler.jedisPool);
                        log.info("【tair 负载均衡】 节点分布 address = {} , 位置 = {}", address, m);
                        virtualCount++;
                    }
                }
                log.info("【tair 负载均衡】 节点个数统计 address = {} ,virtualCount = {}", address, virtualCount);
            }
        }

        public JedisPool select(String key) {
            byte[] digest = md5(key);
            return selectForKey(hash(digest, 0));
        }

        /**
         * 环形数据结构
         */
        private JedisPool selectForKey(long hash) {
            Map.Entry<Long, JedisPool> entry = virtualJedisPool.ceilingEntry(hash);
            if (entry == null) {
                entry = virtualJedisPool.firstEntry();
            }
            log.info("【selectForKey】hash = {}, entry = {}", hash, entry.getValue());
            return entry.getValue();
        }

        /**
         * 划分范围为 ： 无符号正数范围 【0-40亿+】
         */
        private long hash(byte[] digest, int number) {
            return (((long) (digest[3 + number * 4] & 0xFF) << 24)
                    | ((long) (digest[2 + number * 4] & 0xFF) << 16)
                    | ((long) (digest[1 + number * 4] & 0xFF) << 8)
                    | (digest[number * 4] & 0xFF))
                    & 0xFFFFFFFFL;
        }

        private byte[] md5(String value) {
            MessageDigest md5;
            try {
                md5 = MessageDigest.getInstance("MD5");
            } catch (NoSuchAlgorithmException e) {
                throw new IllegalStateException(e.getMessage(), e);
            }
            md5.reset();
            byte[] bytes = value.getBytes(StandardCharsets.UTF_8);
            md5.update(bytes);
            return md5.digest();
        }

    }

}