分布式系统删除zookeeper空节点

在分布式系统结构中，使用ZOOKEEPER里的CuratorFramework控制全局锁，有一个隐藏的问题，生成的全局锁路径释放后，存留了很多空的NODE节点，这些节点数据量很大的时候，会占用很大的磁盘空间，这个可能是ZK版本的一个bug，要是使用完全局锁之后自动帮我们清除掉，就不会有这类问题，针对bug，需要定时清掉空的节点。代码如下，以供记录一下。

1、测试类，测试定时任务手工执行接口，执行成功后，查看一下节点是否真实删除

package xxxxxxxxxxxxxxxxx

import xxxxxxxxxxxx
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import lombok.extern.slf4j.Slf4j;
import org.apache.curator.RetryPolicy;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.recipes.atomic.DistributedAtomicLong;
import org.apache.curator.retry.ExponentialBackoffRetry;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.validation.BindingResult;
import org.springframework.web.bind.annotation.*;

import javax.validation.Valid;
import java.util.HashMap;
import java.util.Map;

@RestController
@Slf4j
@Api(value="test", description="测试")
public class TestController {

	private final CuratorFramework curator;
	private final LockNodeCleanService lockNodeCleanService;



	public TestController(CuratorFramework curator,LockNodeCleanService lockNodeCleanService) {
		this.curator = curator;
		this.lockNodeCleanService =lockNodeCleanService;
	}


	@GetMapping("/internal/zookeeper/listData")
	public void listData(@RequestParam String zkPath) throws Exception {
//		RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);

//		Map<String, Long> result = new HashMap<>();
		for (String string : curator.getChildren().forPath(zkPath)) {
			String tempPath = zkPath + "/" + string;
			log.info("***********:"+tempPath);
		}
	}

	@GetMapping("/internal/zookeeper/remove")
	public void listData() throws Exception {
		lockNodeCleanService.deleteEmptyLockNodes();
	}

}

2、定时任务Servic，自己服务中的节点同时配置了4级和5级两种节点，大家自己服务最好能够统一一种，这样方便处理哈哈。

package xxxxxxxxxxxxxx

import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.framework.recipes.leader.LeaderLatch;
import org.apache.curator.framework.recipes.locks.InterProcessMutex;
import org.apache.curator.retry.RetryNTimes;
import org.apache.zookeeper.data.Stat;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;

import java.util.List;

@Service
@Slf4j
@AllArgsConstructor
public class LockNodeCleanService {
    private final CuratorFramework curator;
    private final LeaderLatch leaderLatch;
    private static final String root = "/locks/quality-test";


    public static void main(String[] args) throws Exception {
        CuratorFramework curator = curator(args[0]);
        InterProcessMutex lock = new InterProcessMutex(curator, "");
        lock.acquire();
        lock.release();
        LockNodeCleanService service = new LockNodeCleanService(curator, null);
        service.deleteEmptyLockNodes();
    }

    private static CuratorFramework curator(String zkAddress) {
        CuratorFramework curator = CuratorFrameworkFactory.
                builder().
                connectString(zkAddress).
                sessionTimeoutMs(1000).
                retryPolicy(new RetryNTimes(3, 1000)).
                build();
        curator.start();
        return curator;
    }

    /**
     * 删除所有空的节点
     */
    @Scheduled(cron = "0 0 4 * * ?")
    public void deleteEmptyLockNodes() {
        if (!leaderLatch.hasLeadership()) {
            log.info("deleteEmptyLockNodes not leader");
            return;
        }
        try {
            //二级目录是否存在
            Stat stat = curator.checkExists().forPath(root);
            if (stat == null) {
                return;
            }

            List<String> folders = curator.getChildren().forPath(root);
            if (CollectionUtils.isEmpty(folders)) {
                return;
            }
            for (String folder : folders) {
                //三级路径
                String folderFullPath = root + "/" + folder;
                try {
                    clearOneFolder(folderFullPath);
                } catch (Exception e) {
                    log.info("error when clear folder: {}", folderFullPath, e);
                }
            }
        } catch (Exception e) {
            log.error("x", e);

        }
    }


    /**
     * 删除一个文件夹下的空节点
     * 一个文件夹就是一种业务对象
     *
     * @param folder
     * @throws Exception
     */
    private void clearOneFolder(String folder) throws Exception {
        for (String nodePath : curator.getChildren().forPath(folder)) {
            //四级路径
            String nodeFullPath = folder + "/" + nodePath;
            List<String> folders = curator.getChildren().forPath(nodeFullPath);
            if (CollectionUtils.isEmpty(folders)) {
                try {
                    //没有子目录直接删除
                    deleteNode(nodeFullPath, curator);
                } catch (Exception e) {
                    log.info("fail to delete: {}", nodeFullPath, e);
                }
                continue;
            }
            for (String path : folders) {
                //删除第五级目录
                String pathFive = nodeFullPath + "/" + path;
                try {
                    deleteNode(pathFive, curator);
                } catch (Exception e) {
                    log.info("fail to delete: {}", pathFive, e);
                }
            }
            try {//删除第四级目录
                deleteNode(nodeFullPath, curator);
            } catch (Exception e) {
                log.info("fail to delete: {}", nodeFullPath, e);
            }




        }
    }


    /**
     * 安全删除由于锁造成的空节点
     * 确保节点存在、没有子节点、没有数据、最近修改时间在一小时前
     *
     * @param path
     * @param curator
     * @throws Exception
     */
    public static void deleteNode(String path, CuratorFramework curator) throws Exception {
        Stat stat = curator.checkExists().forPath(path);
        if (stat == null) {
            log.info(path + "  not exist");
            return;
        }
        if (stat.getNumChildren() > 0) {
            log.info(path + "  has children");
            return;
        }
        if (curator.getData().forPath(path).length > 0) {
            log.info(path + "  has data");
            return;
        }
        if (System.currentTimeMillis() - stat.getMtime() < 3600 * 1000) {
            log.info(path + "  very new");
            return;
        }
        curator.delete().inBackground().forPath(path);
        log.info(path + "  is deleted");
    }

}