异步处理大批量数据

最新推荐文章于 2024-03-21 11:24:29 发布

@郭小茶

最新推荐文章于 2024-03-21 11:24:29 发布

阅读量280

点赞数

分类专栏：成长之路文章标签： java spring boot 异步大批量

本文链接：https://blog.csdn.net/weixin_44570041/article/details/131514613

版权

成长之路专栏收录该内容

14 篇文章 0 订阅

订阅专栏

        List<CompletableFuture<Boolean>> cfs = new ArrayList<>();
        List<Map<String,String>> list1 = new ArrayList<>();
        if (CollectionUtils.isNotEmpty(list)) {
            // 对数据进行批量拆分
            List<List<Map<String,String>>> lists = batchAddRecords(list1);
            List<CompletableFuture<Boolean>> collect = lists.stream().map(lis -> CompletableFuture.supplyAsync(() -> {
                AjaxResult r = save(list1);
                log.info("save -> 保存了{}条数据，返回值为{}", lis.size(), r);
                return Boolean.TRUE;
            }, ForkJoinPool.commonPool())).collect(Collectors.toList());
            cfs.addAll(collect);
        }
        List<Map<String,String>> list2 = new ArrayList<>();
        if (CollectionUtils.isNotEmpty(list2)) {
            List<List<Map<String,String>>> lists = batchAddRecords(list2);
            List<CompletableFuture<Boolean>> collect = lists.stream().map(lis -> CompletableFuture.supplyAsync(() -> {

                AjaxResult r = save(list2);
                log.info("updateBatchInfo -> 保存了{}条数据，返回值为{}", lis.size(), r);
                return Boolean.TRUE;
            }, ForkJoinPool.commonPool())).collect(Collectors.toList());
            cfs.addAll(collect);
        }
        try {
            if (CollectionUtils.isNotEmpty(cfs)) {
                CompletableFuture.allOf(cfs.toArray(new CompletableFuture[cfs.size()])).join();
                for (CompletableFuture<Boolean> cf : cfs) {
                    cf.get();
                }
            }
        } catch (InterruptedException | ExecutionException e) {
            log.error("错误", e);
            throw new IllegalArgumentException("导入失败，请重新导入");
        }

    /**
     * 按每500个一组分割
     *
     * @param list
     * @return
     * @author guoxiaocha
     * @date 2023/3/20 8:57
     */
    private <T> List<List<T>> batchAddRecords(List<T> list) {
        //按每500个一组分割
        int maxValue = 500;
        //计算切分次数
        int limit = (list.size() + maxValue - 1) / maxValue;
        List<List<T>> resultList = new ArrayList<>();
        Stream.iterate(0, n -> n + 1).limit(limit).forEach(i -> {
            resultList.add(list.stream().skip(i * maxValue).limit(maxValue).collect(Collectors.toList()));
        });
        return resultList;
    }