Java导出大批量数据(异步多线程导出篇)

之前文章介绍了

导出文件格式篇(xls,xlsx,csv)https://blog.csdn.net/weixin_56567361/article/details/126640185
分批查询导出篇https://blog.csdn.net/weixin_56567361/article/details/126647979

 本篇介绍下在分批导出时使用线程进行导出

下篇文章将介绍获取@Excel导出字段和字典解析方法

---

 异步多线程+分批查询导出实例

在线程处理时可对分批数据进行线程处理 这里我的分批for循环省略了 展示了对总数据的处理

    private static ThreadPoolExecutor executor = new ThreadPoolExecutor(10, 10, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>());

    
    //Test代表实体类
    //将查询出的list异步处理
    List<testService> cyclesList = testService.selectCycles(test);
            if (cyclesList != null && cyclesList.size() > 0) {
    //使用线程异步处理
    //将数据分组 分成10份 计算每份分的条数(大家可自行调整)
    int remainders = cyclesList.size() % 10;
    int size = remainders > 0 ? cyclesList.size() / 10 + 1 : cyclesList.size() / 10;
    //开辟10个线程异步处理
    CountDownLatch downLatch = new CountDownLatch(10);
    final List<FileVO> finalData = new ArrayList<>();
    for (int j = 0; j < 10; j++) {
        List<Test> testFarmers = cyclesList.subList(j * size, Math.min(cyclesList.size(), j * size + size
        executor.submit(() -> {
            try {
                //数据处理(将每一行字段数据放入List<List<String>> list里 处理方法每人不同)
                FileVO fileVO = testService.dictData(testFarmers , englishTitle, dictMap);
                finalData.add(fileVO);
                //循环一次筛入一次
            } finally {
                downLatch.countDown();
            }
        });
    }
    try {
        downLatch.await();
    } catch (Exception e) {
        e.printStackTrace();
    }
    //将异步处理的list整合
    List<List<String>> getDownloadList = new ArrayList<>();
    for (FileVO vo : finalData) {
        getDownloadList.addAll(vo.getDownloadList());
    }
    //下篇文章将介绍获取@Excel导出字段方法
    PoiUtils.exportCSVFile(表头title, getDownloadList, 循环次数(外面可套for循环), 文件下载绝对路径absolutePath);
    .....

    //在mysql库中测试 30w数据导出约在23s左右
    PoiUtils.outCsvStreamCSV(response, absolutePath);



    /**
     * 上传文件到服务器
     *
     * @param title
     * @param downloadList
     * @param i
     * @param absolutePath
     * @throws IOException
     */
    public static void exportCSVFile(String[] title, List<List<String>> downloadList, int i, String absolutePath) throws IOException {
        //true 拼接导出文件
        BufferedWriter bufferedWriter = new BufferedWriter(new FileWriter(absolutePath, true));
        logger.info("创建文件地址: " + absolutePath);
        //如果是第一次循环 添加表头
        if (i == 0) {
            PoiUtils.writeHead(title, bufferedWriter);
            //另起一行
            bufferedWriter.newLine();
        }
        //循环list中数据 逐个添加
        for (List<String> list : downloadList) {
            CSVFileUtil.writeRow(list, bufferedWriter);
            bufferedWriter.newLine();
        }
        bufferedWriter.close();
    }


    /**
     * 分割csv文件传浏览器(适用xlxs文件)
     *
     * @param response
     * @param absolutePath
     * @throws IOException
     */
    public static void outCsvStreamCSV(HttpServletResponse response, String absolutePath) throws IOException {
        java.io.OutputStream out = response.getOutputStream();
        byte[] b = new byte[10240];
        java.io.File fileLoad = new java.io.File(absolutePath);
        response.reset();
        response.setContentType("application/csv");
        response.setHeader("content-disposition", "attachment; filename=" + URLEncoder.encode("export.csv", "UTF-8"));
        java.io.FileInputStream in = new java.io.FileInputStream(fileLoad);
        int n;
        //为了保证excel打开csv不出现中文乱码
        out.write(new byte[]{(byte) 0xEF, (byte) 0xBB, (byte) 0xBF});
        while ((n = in.read(b)) != -1) {
            //每次写入out1024字节
            out.write(b, 0, n);
        }
        in.close();
        out.close();
    }

自定义FileVo

import java.util.List;

/**
 * 文件类
 */
public class FileVO {
    private String name;
    private int type;

    private List<String[]> listString;
    private List<List<String>> downloadList;
    private String[] title;

    public List<String[]> getListString() {
        return listString;
    }

    public void setListString(List<String[]> listString) {
        this.listString = listString;
    }

    public List<List<String>> getDownloadList() {
        return downloadList;
    }

    public void setDownloadList(List<List<String>> downloadList) {
        this.downloadList = downloadList;
    }

    public String[] getTitle() {
        return title;
    }

    public void setTitle(String[] title) {
        this.title = title;
    }

    public String getName() {
        return name;
    }

    public void setName(String name) {
        this.name = name;
    }

    public int getType() {
        return type;
    }

    public void setType(int type) {
        this.type = type;
    }

    @Override
    public String toString() {
        return "FileVO{" +
                "name='" + name + '\'' +
                ", type=" + type +
                '}';
    }
}

希望能帮助到大家

大家根据需求调整代码 根据源码多测试

最后有遇到什么问题可以留言告诉我哦 欢迎评论区讨论😉