大数据量导出，CountDownLatch线程交互问题

需求

导出数据量过大时会造成接口超时异常，所以我们希望分页进行导出。

解决过程

核心方法

/**
     * 查询需要导出excel数据
     * @param query 查询条件
     * @param url 回掉地址
     * @return 查询结果
     */
    private List<Map<String, Object>> batchRequest(Map<String, Object> query, String url) {
        Integer total = (Integer) query.get(ExcelConstant.TOTAL);
        log.info("总条目数为：{}", total);
        int operations = total / LIMIT + (total % LIMIT > 0 ? 1 : 0);
        List<Map<String, Object>> resultList = new CopyOnWriteArrayList<>();
        CountDownLatch countDownLatch = new CountDownLatch(operations);
        for (int i = 1; i <= operations ; i++) {
            Integer page = i;
            taskExecutor.execute(() -> {
                query.put("page", page);
                query.put("limit", LIMIT);
                Result<PagePagination<Map<String, Object>>> pagePaginationResult = null;
                try {
                // 对数据进行请求
                    pagePaginationResult = feignClient.pageQuery(new URI(url), query);
                } catch (URISyntaxException e) {
                    log.error("导出excel回掉接口错误，参数为：url:{},param:{}", url, query);
                    e.printStackTrace();
                }
                PagePagination<Map<String, Object>> pagePagination = ResultDataUtils.getInstance(pagePaginationResult).getDataOrException();
                log.info(Thread.currentThread().getName() + "查询到{}条数据,此时总集合数据为：{}", pagePagination.getList().size(), resultList.size());
                resultList.addAll(pagePagination.getList());
                log.info(Thread.currentThread().getName() + "addAll之后总集合数据为：{}", resultList.size());
                countDownLatch.countDown();
            });

        }
        try {
            countDownLatch.await();
            log.info(Thread.currentThread().getName() + "countDownLatch.getCount() 为：{}", countDownLatch.getCount());
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        return resultList;
    }

问题一

要么每个线程请求到的数据都是500条（代码中LIMIT常量等于500），要么每个线程请求到的数据都是417（一共为1417条数据）。打印日志后发现三个线程page要么是1要么是3

问题一调整之后代码

启用原子整型（AtomicInteger）解决读取未来得及变化的page变量问题

 /**
     * 查询需要导出excel数据
     * @param query 查询条件
     * @param url 回掉地址
     * @return 查询结果
     */
    private List<Map<String, Object>> batchRequest(Map<String, Object> query, String url) {
        Integer total = (Integer) query.get(ExcelConstant.TOTAL);
        log.info("总条目数为：{}", total);
        int operations = total / LIMIT + (total % LIMIT > 0 ? 1 : 0);
        AtomicInteger atomicInteger = new AtomicInteger(operations);
        List<Map<String, Object>> resultList = new CopyOnWriteArrayList<>();
        CountDownLatch countDownLatch = new CountDownLatch(operations);
        for (int i = 1; i <= operations ; i++) {
            Integer page = i;
            taskExecutor.execute(() -> {
                query.put("page", operations - atomicInteger.decrementAndGet());
                query.put("limit", LIMIT);
                Result<PagePagination<Map<String, Object>>> pagePaginationResult = null;
                try {
//                    log.info("查询条件为页码：{}", copyQuery.get("page"));
                    pagePaginationResult = feignClient.pageQuery(new URI(url), query);
                    log.info(Thread.currentThread().getName() + "查询条件为页码：{}", query.get("page"));
                } catch (URISyntaxException e) {
                    log.error("导出excel回掉接口错误，参数为：url:{},param:{}", url, query);
                    e.printStackTrace();
                }
                PagePagination<Map<String, Object>> pagePagination = ResultDataUtils.getInstance(pagePaginationResult).getDataOrException();
                log.info(Thread.currentThread().getName() + "查询到{}条数据,此时总集合数据为：{}", pagePagination.getList().size(), resultList.size());
                resultList.addAll(pagePagination.getList());
                log.info(Thread.currentThread().getName() + "addAll之后总集合数据为：{}", resultList.size());
                countDownLatch.countDown();
            });

        }
        try {
            countDownLatch.await();
            log.info(Thread.currentThread().getName() + "countDownLatch.getCount() 为：{}", countDownLatch.getCount());
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        return resultList;
    }

问题二

依旧存在问题一的问题

问题二原因

变量query存在堆中，虽然在query.put(“page”, operations - atomicInteger.decrementAndGet());的时候设置了页码，但是还是存在请求之前数据被其他线程修改了的问题

问题二调整之后代码

将query在堆中变量为每个线程单独copy一份，避免其他线程对相关参数的修改

    /**
     * 查询需要导出excel数据
     * @param query 查询条件
     * @param url 回掉地址
     * @return 查询结果
     */
    private List<Map<String, Object>> batchRequest(Map<String, Object> query, String url) {
        Integer total = (Integer) query.get(ExcelConstant.TOTAL);
        log.info("总条目数为：{}", total);
        int operations = total / LIMIT + (total % LIMIT > 0 ? 1 : 0);
        AtomicInteger atomicInteger = new AtomicInteger(operations);
        List<Map<String, Object>> resultList = new CopyOnWriteArrayList<>();
        CountDownLatch countDownLatch = new CountDownLatch(operations);
        for (int i = 1; i <= operations ; i++) {
            Integer page = i;
            taskExecutor.execute(() -> {
                Map<String, Object> copyQuery = new HashMap<>(query.size());
                BeanUtil.copyProperties(query, copyQuery, CopyOptions.create());
                copyQuery.put("page", operations - atomicInteger.decrementAndGet());
                copyQuery.put("limit", LIMIT);
                Result<PagePagination<Map<String, Object>>> pagePaginationResult = null;
                try {
                    pagePaginationResult = feignClient.pageQuery(new URI(url), copyQuery);
                    log.info(Thread.currentThread().getName() + "查询条件为页码：{}", copyQuery.get("page"));
                } catch (URISyntaxException e) {
                    log.error("导出excel回掉接口错误，参数为：url:{},param:{}", url, copyQuery);
                    e.printStackTrace();
                }
                PagePagination<Map<String, Object>> pagePagination = ResultDataUtils.getInstance(pagePaginationResult).getDataOrException();
                log.info(Thread.currentThread().getName() + "查询到{}条数据,此时总集合数据为：{}", pagePagination.getList().size(), resultList.size());
                resultList.addAll(pagePagination.getList());
                log.info(Thread.currentThread().getName() + "addAll之后总集合数据为：{}", resultList.size());
                countDownLatch.countDown();
            });

        }
        try {
            countDownLatch.await();
            log.info(Thread.currentThread().getName() + "countDownLatch.getCount() 为：{}", countDownLatch.getCount());
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        return resultList;
    }