Nacos——Distro一致性协议

Nacos——Distro一致性协议

1. 理论

一致性一直都是分布式系统中绕不开的话题。根据CAP中，要么CP(保证强一致性牺牲可用性)，要么AP(最终一致性来保证可用性)，在市面上也有几种一致性算法，像Paxos，Raft，Zookeeper的ZAB等。而Nacos实现了AP和CP，对非持久化实例实现了基于CP的Distro协议，那接下来就看看这个协议的工作流程。

2. 调试环境

由于需要跟踪源码并且在集群模式下，所以这里设计了这样的调试环境

1. 将Nacos源码克隆，github地址：https://github.com/alibaba/nacos，这里用的是1.4.2版本；

2. 本地数据库执行distribution/conf目录下的nacos-mysql.sql创建nacos数据库；

3. 修改console模块下的application.properties，将端口变成读取环境变量(方便一份代码起多个实例)，及修改数据库配置；

   server.port=${port}
   
   spring.datasource.platform=mysql
   db.num=1
   db.url.0=jdbc:mysql://127.0.0.1:3306/nacos?characterEncoding=utf8&connectTimeout=1000&socketTimeout=3000&autoReconnect=true&useUnicode=true&useSSL=false&serverTimezone=UTC
   db.user.0=root
   db.password.0=root
   

4. IDEA启动多个实例，启动主类在console目录下的Nacos.java，添加环境变量和jvm参数；

   环境变量
   port=8848
   
   jvm参数
   -Dnacos.home=D:/nacos-home/nacos-8848 -Dnacos.standalone=false -DembeddedStorage=true
   

   注意：指定nacos home目录时也要按不同端口号区分目录

5. 在nacos home路径下新建conf目录，创建cluster.conf文件。其他节点的home目录同样操作。

   # 集群中实例地址
   主机ip:8848
   主机ip:8849
   主机ip:8850
   

3. 源码分析

节点启动全量同步其他节点数据

服务端节点启动时，初始化DistroProtocol类，在构造函数中开启同步其他服务端节点数据任务

DistroProtocol

public DistroProtocol(ServerMemberManager memberManager, DistroComponentHolder distroComponentHolder,
          DistroTaskEngineHolder distroTaskEngineHolder, DistroConfig distroConfig) {
   
    this.memberManager = memberManager;
    this.distroComponentHolder = distroComponentHolder;
    this.distroTaskEngineHolder = distroTaskEngineHolder;
    this.distroConfig = distroConfig;
    // 启动当前节点时同步其他节点的全量数据
    startDistroTask();
}

private void startDistroTask() {
   
    if (EnvUtil.getStandaloneMode()) {
   
        isInitialized = true;
        return;
    }
    // 开启节点间心跳检测
    startVerifyTask();
    // 开启同步其他节点数据
    startLoadTask();
}

/**
 * 同步其他节点数据
 */
private void startLoadTask() {
   
    // 同步回调
    DistroCallback loadCallback = new DistroCallback() {
   
        @Override
        public void onSuccess() {
   
            isInitialized = true;
        }

        @Override
        public void onFailed(Throwable throwable) {
   
            isInitialized = false;
        }
    };
    // 将同步数据任务放入线程池中执行
    GlobalExecutor.submitLoadDataTask(
        new DistroLoadDataTask(memberManager, distroComponentHolder, distroConfig, loadCallback));
}

同步其他节点数据任务类DistroLoadDataTask.run()

@Override
public void run() {
   
    try {
   
        // 从其他节点加载数据
        load();
        // 如果加载数据不成功则开启新的线程继续去拉取全量数据直到加载成功
        if (!checkCompleted()) {
   
            GlobalExecutor.submitLoadDataTask(this, distroConfig.getLoadDataRetryDelayMillis());
        } else {
   
            loadCallback.onSuccess();
            Loggers.DISTRO.info("[DISTRO-INIT] load snapshot data success");
        }
    } catch (Exception e) {
   
        loadCallback.onFailed(e);
        Loggers.DISTRO.error("[DISTRO-INIT] load snapshot data failed. ", e);
    }
}

private void load() throws Exception {
   
    // 除自身之外没有其他节点，则休眠一秒，等待其他节点启动
    while (memberManager.allMembersWithoutSelf().isEmpty()) {
   
        Loggers.DISTRO.info("[DISTRO-INIT] waiting server list init...");
        TimeUnit.SECONDS.sleep(1);
    }
    // 等待数据类型初始化完毕
    while (distroComponentHolder.getDataStorageTypes().isEmpty()) {
   
        Loggers.DISTRO.info("[DISTRO-INIT] waiting distro data storage register...");
        TimeUnit.SECONDS.sleep(1);
    }
    // 加载每个数据类型
    for (String each : distroComponentHolder.getDataStorageTypes()) {
   
        if (!loadCompletedMap.containsKey(each) || !loadCompletedMap.get(each)) {
   
            loadCompletedMap.put(each, loadAllDataSnapshotFromRemote(each));
        }
    }
}

/**
 * 从远程节点拉取全量数据
 */
private boolean loadAllDataSnapshotFromRemote(String resourceType) {
   
    // 用于远程拉取数据
    DistroTransportAgent transportAgent = distroComponentHolder.findTransportAgent(resourceType);
    // 用于处理数据
    DistroDataProcessor dataProcessor = distroComponentHolder.findDataProcessor(resourceType);
    if (null == transportAgent || null == dataProcessor) {
   
        Loggers.DISTRO.warn("[DISTRO-INIT] Can't find component for type {}, transportAgent: {}, dataProcessor: {}", resourceType, transportAgent, dataProcessor);
        return false;
    }
    // 循环每个节点
    for (Member each : memberManager.allMembersWithoutSelf()) {
   
        try {
   
            Loggers.DISTRO.info("[DISTRO-INIT] load snapshot {} from {}", resourceType, each.getAddress());
            // 调用远程节点API GET /distro/datums拉取数据
            DistroData distroData = transportAgent.getDatumSnapshot(each.getAddress());
            // 处理数据
            boolean result = dataProcessor.processSnapshot(distroData);
            Loggers.DISTRO
                .info("[DISTRO-INIT] load snapshot {} from {} result: {}", resourceType, each.getAddress(), result);
            // 如果处理成功则返回，不再循环其他节点
            if (result) {
   
                return true;
            }
        } catch (Exception e) {
   
            Loggers.DISTRO.error("[DISTRO-INIT] load snapshot {} from {} failed.", resourceType, each.getAddress(), e);
        }
    }
    return false;
}

处理数据DistroConsistencyServiceImpl.processSnapshot()

@Override
public boolean processSnapshot(DistroData distroData) {
   
    try {
   
        return processData(distroData.getContent());
    } catch (Exception e) {
   
        return false;
    }
}

/**
 * 处理其他节点返回的数据
 */
private boolean processData(byte[] data) throws Exception {
   
    if (data.length > 0) {
   
        // 反序列化数据为对象
        Map<String, Datum<Instances>> datumMap = serializer.deserializeMap(data, Instances.class);

        for (Map.Entry<String, Datum<Instances>> entry : datumMap.entrySet()) {
   
            // 数据存入DataStore的dataMap中
            dataStore.put(entry.getKey(), entry.getValue());

            if (!listeners.containsKey(entry.getKey())) {
   
                // pretty sure the service not exist:
                if (switchDomain.isDefaultInstanceEphemeral()) {
   
                    // 创建空Service
                    Loggers.DISTRO.info("creating service {}", entry.getKey());
                    Service service = new Service();
                    String serviceName = KeyBuilder.getServiceName(entry.getKey());
                    String namespaceId = KeyBuilder.getNamespace(entry.getKey());
                    service.setName(serviceName);
                    service.setNamespaceId(namespaceId);
                    service.setGroupName(Constants.DEFAULT_GROUP);
                    // now validate the service. if failed, exception will be thrown
                    service.setLastModifiedMillis(System.currentTimeMillis());
                    service.recalculateChecksum();

                    // key=com.alibaba.nacos.naming.domains.meta.的listener必须不能为空
                    RecordListener listener = listeners.get(KeyBuilder.SERVICE_META_KEY_PREFIX).peek();
                    if (Objects.isNull(listener)) {
   
                        return false;
                    }
                    // ServiceManager.onChange
                    listener.onChange(KeyBuilder.buildServiceMetaKey(namespaceId, serviceName), service);
                }
            }
        }

        for (Map.Entry<String, Datum<Instances>> entry : datumMap.entrySet()) {
   

            if (!listeners.containsKey(entry.getKey())) {
   
                // Should not happen:
                Loggers.DISTRO.warn("listener of {} not found.", entry.getKey());
                continue;
            }

            try {
   
                for (RecordListener listener : listeners.get(entry.getKey())) {
   
                    // 调用指定Service.onChange()
                    listener.onChange(entry.getKey(), entry.getValue().value);
                }
            } catch (Exception e) {
   
                Loggers.DISTRO.error("[NACOS-DISTRO] error while execute listener of key: {}", entry.getKey(), e);
                continue;
            }

            // 由于对Service进行了修改所以要更新DataStore
            dataStore.put(entry.getKey(), entry.getValue());
        }
    }
    return true;
}

处理数据步骤：

1. 存入到DataStore.dataMap
2. 调用ServiceManager.onChange()
3. 调用Service.onChange()
4. 更新DataStore.dataMap

ServiceManager.onChange()

作用：

1. 将Service添加到ServiceManager.serviceMap变量
2. 将Service做为RecordListener添加到DistroConsistencyServiceImpl.listeners变量(用来新增、移除实例时，发送通知给订阅者)

@Override
public void onChange(String key, Service service) throws Exception {
   
    try {
   
        if (service == null) {
   
            Loggers.SRV_LOG.warn("received empty push from raft, key: {}", key);
            return;
        }

        if (StringUtils.isBlank(service.getNamespaceId())) {
   
            service.setNamespaceId(Constants.DEFAULT_NAMESPACE_ID);
        }

        Loggers.RAFT.info("[RAFT-NOTIFIER] datum is changed, key: {}, value: {}", key, service);

        // 从serviceMap中获取Service
        Service oldDom = getService(service.getNamespaceId(), service.getName());

        // 旧Service不为空，则进行数据更新，并重新加入到DistroConsistencyServiceImpl.listeners中
        if (oldDom != null) {
   
            oldDom.update(service);
            // re-listen to handle the situation when the underlying listener is removed:
            consistencyService
                .listen(KeyBuilder.buildInstanceListKey(service.getNamespaceId(), service.getName(), true), oldDom);
            consistencyService
                .listen(KeyBuilder.buildInstanceListKey(service.getNamespaceId(), service.getName(), false), oldDom);
        } else {
   
            // 旧Service不为空，加入到serviceMap和加入到DistroConsistencyServiceImpl.listeners中
            putServiceAndInit(service);
        }
    } catch (Throwable e) {
   
        Loggers.SRV_LOG.error("[NACOS-SERVICE] error while processing service update", e);
    }
}

private void putServiceAndInit(Service service) throws NacosException {
   
    // service添加到serviceMap中
    putService(service);
    service = getService(service.getNamespaceId(), service.getName());
    // 开启心跳检测
    service.init();
    // 添加到DistroConsistencyServiceImpl.listeners中
    consistencyService
        .listen(KeyBuilder.buildInstanceListKey(service.getNamespaceId(), service.getName(), true), service);
    consistencyService
        .listen(KeyBuilder.buildInstanceListKey(service.getNamespaceId