easyExcel和poi多线程实现百万级数据量导出DEMO

syfjava

已于 2023-06-06 14:32:18 修改

阅读量847

点赞数 4

分类专栏： CompletableFuture 异步编排与实际代码展示文章标签： java 开发语言

于 2023-06-05 19:10:19 首次发布

本文链接：https://blog.csdn.net/javaxueba/article/details/131003291

版权

CompletableFuture 异步编排与实际代码展示专栏收录该内容

5 篇文章 0 订阅

订阅专栏

基于easyExcel方式导出

controller层接口

   @GetMapping(value = "/exportEasyExcel")
    public  void exportEasyExcel(HttpServletResponse response,Integer total,Integer limit) {
        long start = System.currentTimeMillis();
        System.out.println("-----------任务执行开始-----------");
        // 设置响应内容
        response.setContentType("application/vnd.ms-excel");
        response.setCharacterEncoding("UTF-8");// 防止下载的文件名字乱码
        try {
            // 文件以附件形式下载
            response.setHeader("Content-disposition",
                    "attachment;filename=down_" +URLEncoder.encode(System.currentTimeMillis() + ".xlsx", "utf-8"));
            List<UserEntity> userEntityList = userService.selectAll(total,limit);
            long start1 = System.currentTimeMillis();
            EasyExcel.write(response.getOutputStream(), UserEntity.class).excelType(ExcelTypeEnum.XLSX).sheet("文件下载")
                    .doWrite(userEntityList);
            long end = System.currentTimeMillis();
            System.out.println("导出execl文件生成总耗时：  " + (end - start1) + "ms");
            System.out.println("本次导出execl任务执行完毕,共消耗：  " + (end - start) + "ms");
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }

selectAll() 多线程查询的具体方法

    public List<UserEntity> selectAll(Integer total,Integer limit) {
        long start = System.currentTimeMillis();
        List<MyCallableTsk> myCallableTsks = new ArrayList<>();
        // 计算出多少页，即循环次数
        int count = total / limit + (total % limit > 0 ? 1 : 0);
        System.out.println("本次任务量: "+count);
        //定义线程计数器
        CountDownLatch cd = new CountDownLatch(count);
        //生成对应的线程
        for (int i = 1; i <= count; i++) {
            myCallableTsks.add(new MyCallableTsk(i,limit,cd));
        }
        List<UserEntity> userEntityList = new ArrayList<>();
        try {
            List<Future<List<UserEntity>>> futures = threadPoolExecutor.invokeAll(myCallableTsks);
            for (Future<List<UserEntity>> future : futures) {
                userEntityList.addAll(future.get());
            }
        } catch (InterruptedException | ExecutionException e) {
            e.printStackTrace();
        }
        long end = System.currentTimeMillis();
        System.out.println("主线程：" + Thread.currentThread().getName() + " , 导出指定数据成功 , 共导出数据：" + userEntityList.size() + " ,查询数据任务执行完毕共消耗时 ：" + (end - start) + "ms");
        return userEntityList;
    }

MyCallableTsk 实现 Callable接口

   class MyCallableTsk implements Callable<List<UserEntity>>{
        private UserMapper userMapper;
        private CountDownLatch cd;

        private Integer pageNum;
        private Integer pageSize;

        public MyCallableTsk(Integer pageNum,Integer pageSize,CountDownLatch cd){
            this.pageNum =pageNum;
            this.pageSize =pageSize;
            this.cd=cd;
        }

        //重写线程任务类的call()方法，这个方法可以直接返回执行的结果。
        @Override
        public List<UserEntity> call() throws Exception {
            System.out.println("线程：" + Thread.currentThread().getName() + " , 开始读取数据------");
            long start = System.currentTimeMillis();
            userMapper = (UserMapper) SpringUtil.getBean("userMapper");
            PageHelper.startPage(pageNum,pageSize);
            List<UserEntity> userEntityList = UserServiceImpl.this.userMapper.selectAll();
            System.out.println("线程：" + Thread.currentThread().getName() + " , 读取数据  "+userEntityList.size()+",页数:"+pageNum+ "耗时 ：" + (System.currentTimeMillis() - start)+ "ms");
            cd.countDown();
            System.out.println("剩余任务数  ===========================> " + cd.getCount());
            return userEntityList;
        }
    }

线程池配置类

@Configuration
@EnableAsync
public class AsyncTaskPoolConfig {

    @Bean("taskExecutor")
    public ThreadPoolExecutor taskExecutor() {
        //获取最大线程数
        int i = Runtime.getRuntime().availableProcessors();
        System.out.println("系统最大线程数：" + i);
        ThreadPoolExecutor threadPoolExecutor = new ThreadPoolExecutor(i,
                i+1, 5, TimeUnit.SECONDS, new LinkedBlockingDeque<>(),new NamedThreadFactory("execl导出线程池"));
        System.out.println("execl导出线程池初始化完毕-------------");
        return threadPoolExecutor;
    }
}

NamedThreadFactory 线程池工厂

public class NamedThreadFactory implements ThreadFactory {

        private final AtomicInteger poolNumber = new AtomicInteger(1);

        private final ThreadGroup threadGroup;

        private final AtomicInteger threadNumber = new AtomicInteger(1);

        public  final String namePrefix;

        NamedThreadFactory(String name){
            SecurityManager s = System.getSecurityManager();
            threadGroup = (s != null) ? s.getThreadGroup() :
                    Thread.currentThread().getThreadGroup();
            if (null==name || "".equals(name.trim())){
                name = "pool";
            }
            namePrefix = name +"-"+
                    poolNumber.getAndIncrement() +
                    "-thread-";
        }

        @Override
        public Thread newThread(Runnable r) {
            Thread t = new Thread(threadGroup, r,
                    namePrefix + threadNumber.getAndIncrement(),
                    0);
            if (t.isDaemon())
                t.setDaemon(false);
            if (t.getPriority() != Thread.NORM_PRIORITY)
                t.setPriority(Thread.NORM_PRIORITY);
            return t;
        }
    }

UserEntity实体类

@Data
@TableName(value = "user")//指定表名
public class UserEntity implements Serializable {

    private static final long serialVersionUID = -5644799954031156649L;

    @TableId(value = "id", type = IdType.AUTO)//指定自增策略
    @ExcelProperty(value = "主键ID" ,index = 0)
    private String id;

    @TableField("name")
    @ExcelProperty(value = "姓名" ,index = 1)
    private String name;

    @TableField("phone")
    @ExcelProperty(value = "手机" ,index = 2)
    private String phone;

    @TableField("ceate_by")
    @ExcelProperty(value = "创建人" ,index = 3)
    private String ceateBy;

    @TableField("remark")
    @ExcelProperty(value = "备注" ,index = 4)
    private String remark;

    /*@TableField("birthday")
    @ExcelProperty(value = "生日" ,index = 5)
    private Date birthday;*/

    @TableField("test1")
    @ExcelProperty(value = "测试数据1" ,index = 6)
    private String test1;

    @TableField("test2")
    @ExcelProperty(value = "测试数据2" ,index = 7)
    private String test2;

    @TableField("test3")
    @ExcelProperty(value = "测试数据3" ,index = 8)
    private String test3;

    @TableField("test4")
    @ExcelProperty(value = "测试数据4" ,index = 9)
    private String test4;

    @TableField("test5")
    @ExcelProperty(value = "测试数据5" ,index = 10)
    private String test5;

    @TableField("test6")
    @ExcelProperty(value = "测试数据6" ,index = 11)
    private String test6;

    @TableField("test7")
    @ExcelProperty(value = "测试数据7" ,index = 12)
    private String test7;

    @TableField("test8")
    @ExcelProperty(value = "测试数据8" ,index = 13)
    private String test8;

    @TableField("test9")
    @ExcelProperty(value = "测试数据9" ,index = 14)
    private String test9;

    @TableField("test10")
    @ExcelProperty(value = "测试数据10" ,index = 15)
    private String test10;

    @TableField("test11")
    @ExcelProperty(value = "测试数据11" ,index = 16)
    private String test11;

    @TableField("test12")
    @ExcelProperty(value = "测试数据12" ,index =17)
    private String test12;

    @TableField("test13")
    @ExcelProperty(value = "测试数据13" ,index = 18)
    private String test13;

    @TableField("test14")
    @ExcelProperty(value = "测试数据14" ,index = 19)
    private String test14;

    @TableField("test15")
    @ExcelProperty(value = "测试数据15" ,index = 20)
    private String test15;

    @TableField("test16")
    @ExcelProperty(value = "测试数据16" ,index = 21)
    private String test16;

    @TableField("test17")
    @ExcelProperty(value = "测试数据17" ,index = 22)
    private String test17;

    @TableField("test18")
    @ExcelProperty(value = "测试数据18" ,index = 23)
    private String test18;

    @TableField("test19")
    @ExcelProperty(value = "测试数据19" ,index = 24)
    private String test19;

    @TableField("test20")
    @ExcelProperty(value = "测试数据20" ,index = 25)
    private String test20;

    @TableField("test21")
    @ExcelProperty(value = "测试数据21" ,index = 26)
    private String test21;
}

基于 poi 导出数据

controller代码

userService.selectAll(total,limit); 多线程查询数据同上selectAll() 具体方法

@GetMapping(value = "/exportEasyExcelV4")
    public  void exportEasyExcelV4(HttpServletResponse response,Integer total,Integer limit) throws IOException {
        long start = System.currentTimeMillis();
        System.out.println("-----------任务执行开始-----------");
        // 设置响应内容
        response.setContentType("application/vnd.ms-excel");
        response.setCharacterEncoding("UTF-8");// 防止下载的文件名字乱码
        // 文件以附件形式下载
        response.setHeader("Content-disposition",
                "attachment;filename=down_" +URLEncoder.encode(System.currentTimeMillis() + ".xlsx", "utf-8"));
        try {
            String sheetName = "业务清单";
            String[] title = new String[]{
                    "主键ID"
                    ,"姓名"
                    ,"手机"
                    ,"创建人"
                    ,"备注"
                    ,"测试数据1"
                    ,"测试数据2"
                    ,"测试数据3"
                    ,"测试数据4"
                    ,"测试数据5"
                    ,"测试数据6"
                    ,"测试数据7"
                    ,"测试数据8"
                    ,"测试数据9"
                    ,"测试数据10"
                    ,"测试数据11"
                    ,"测试数据12"
                    ,"测试数据13"
                    ,"测试数据14"
                    ,"测试数据15"
                    ,"测试数据16"
                    ,"测试数据17"
                    ,"测试数据18"
                    ,"测试数据19"
                    ,"测试数据20"
                    ,"测试数据21"
            };
            List<List<String>> data1 = new ArrayList<List<String>>();
            List<UserEntity> userEntityList = userService.selectAll(total,limit);
            for (UserEntity userEntity : userEntityList) {
                List<String> rowData = new ArrayList<>();
                rowData.add(userEntity.getId()); //归属机构
                rowData.add(userEntity.getName()); //姓名
                rowData.add(userEntity.getPhone()); //姓名java'
                rowData.add(userEntity.getCeateBy()); //姓名
                rowData.add(userEntity.getRemark()); //姓名
                rowData.add(userEntity.getTest1()); //姓名
                rowData.add(userEntity.getTest2()); //姓名
                rowData.add(userEntity.getTest3()); //姓名
                rowData.add(userEntity.getTest4()); //姓名
                rowData.add(userEntity.getTest5()); //姓名
                rowData.add(userEntity.getTest6()); //姓名
                rowData.add(userEntity.getTest7()); //姓名
                rowData.add(userEntity.getTest8()); //姓名
                rowData.add(userEntity.getTest9()); //姓名
                rowData.add(userEntity.getTest10()); //姓名
                rowData.add(userEntity.getTest11()); //姓名
                rowData.add(userEntity.getTest12()); //姓名
                rowData.add(userEntity.getTest13()); //姓名
                rowData.add(userEntity.getTest14()); //姓名
                rowData.add(userEntity.getTest15()); //姓名
                rowData.add(userEntity.getTest16()); //姓名
                rowData.add(userEntity.getTest17()); //姓名
                rowData.add(userEntity.getTest18()); //姓名
                rowData.add(userEntity.getTest19()); //姓名
                rowData.add(userEntity.getTest20()); //姓名
                rowData.add(userEntity.getTest21()); //姓名
                data1.add(rowData);
            }
            long start1 = System.currentTimeMillis();
            SXSSFWorkbook wb = PoiSXSSFWorkbookExcel.getSxssfwbExcel(sheetName,title, data1);
            long end = System.currentTimeMillis();
            System.out.println("导出execl文件生成总耗时：  " + (end - start1) + "ms");
            ServletOutputStream output = response.getOutputStream();
            wb.write(output);
            output.flush();
            output.close();
            System.out.println("本次导出execl任务执行完毕,共"+userEntityList.size()+"条数据,共消耗：  " + (end - start) + "ms");
        } catch (Exception e) {
            throw new RuntimeException(e);
        }

    }

PoiSXSSFWorkbookExcel 工具类代码

public class PoiSXSSFWorkbookExcel {

/**
 * 导出到单个sheet页中
 * @param title
 *            标题
 * @param List<List<String>> result
 *            导出数据内容
 * @return wb
 *            SXSSFWorkbook对象
 * @throws IOException
 * 
 */

public static SXSSFWorkbook getSxssfwbExcel(String sheetTitle,String[] title, List<List<String>> result) {
	SXSSFWorkbook wb = new SXSSFWorkbook();
	//int sheetNum = 0;// 记录额外创建的sheet数量
	Sheet sheet = wb.createSheet(sheetTitle);
	// wb.setSheetName(sheetNum, sheetTitle+sheetNum);
	int rownum = 0;
	Row row = sheet.createRow(rownum);
	
	// 设置并获取到需要的样式
    XSSFCellStyle xssfCellStyleHeader = getAndSetXSSFCellStyleHeader(wb);
	
	Cell cell;
	// 创建标题,此时row=0,即第一行
	for (int j = 0; j < title.length; j++) {
		cell = row.createCell(j);
		cell.setCellValue(title[j]);
		cell.setCellStyle(xssfCellStyleHeader);
	}

	// 遍历集合数据，创建excel内容,产生数据行
	if (result != null) {
		int index = 1;
		List<String> m = null;
		for (int i = 0; i < result.size(); i++) {
			row = sheet.createRow(index);
			int cellIndex = 0;
			m = result.get(i);
			for (String str : m) {
				row.createCell((short) cellIndex).setCellValue(str);
				cellIndex++;
			}
			
			index++;
		}
	}
	
	
	return wb;
}



/**
 * 每个sheet页导出不同内容到多个sheet页中，每个sheet页表头内容不相同
 * 
 * @param wb
 *            SXSSFWorkbook对象
 * @param sheetNum
 *            sheet页
 * @param sheetTitle
 *            sheet页名称
 * @param title
 *            标题
 * @param List<List<String>> result
 *            导出数据内容
 * @return wb
 *            SXSSFWorkbook对象
 * @throws IOException
 * 
 */

public static SXSSFWorkbook getSxssfwbManyDiffSheets(SXSSFWorkbook wb, int sheetNum, String sheetTitle,
		String[] title, List<List<String>> result) {
	
	Sheet sheet = wb.createSheet();
	wb.setSheetName(sheetNum, sheetTitle);
	int rownum = 0;
	Row row = sheet.createRow(rownum);
	
	// 设置并获取到需要的样式
    XSSFCellStyle xssfCellStyleHeader = getAndSetXSSFCellStyleHeader(wb);
	
	Cell cell;
	// 创建标题,此时row=0,即第一行
	for (int j = 0; j < title.length; j++) {
		cell = row.createCell(j);
		cell.setCellValue(title[j]);
		cell.setCellStyle(xssfCellStyleHeader);
	}

	// 遍历集合数据，创建excel内容,产生数据行
	if (result != null) {
		int index = 1;
		List<String> m = null;
		for (int i = 0; i < result.size(); i++) {
			row = sheet.createRow(index);
			int cellIndex = 0;
			m = result.get(i);
			for (String str : m) {
				row.createCell((short) cellIndex).setCellValue(str);
				cellIndex++;
			}
			
			index++;
		}
	}
	
	
	return wb;
}




/**
 * 每个sheet页导出相同内容到多个sheet页中，每个sheet页表头内容相同
 * 每个sheet页目前总数设置为n,这里设置n=60000
 * @param title
 *            标题
 * @param List<List<String>> result
 *            导出数据内容
 * @return wb
 *            SXSSFWorkbook对象
 * @throws IOException
 * 
 */

public static SXSSFWorkbook getSxssfwbManySameSheets(String sheetTitle,String[] title, List<List<String>> result) {
	SXSSFWorkbook wb = new SXSSFWorkbook();
	int sheetNum = 0;// 记录额外创建的sheet数量
	Sheet sheet = wb.createSheet(sheetTitle + sheetNum);
	// wb.setSheetName(sheetNum, sheetTitle+sheetNum);
	int rownum = 0;
	Row row = sheet.createRow(rownum);
	
	// 设置并获取到需要的样式
    XSSFCellStyle xssfCellStyleHeader = getAndSetXSSFCellStyleHeader(wb);
	
	Cell cell;
	// 创建标题,此时row=0,即第一行
	for (int j = 0; j < title.length; j++) {
		cell = row.createCell(j);
		cell.setCellValue(title[j]);
		cell.setCellStyle(xssfCellStyleHeader);
	}

	// 遍历集合数据，创建excel内容,产生数据行
	if (result != null) {
		List<String> m = null;
		for (int i = 0; i < result.size(); i++) {
			if ((i + 1) % 60000 == 0) {
				sheetNum++;
				sheet = wb.createSheet(sheetTitle + sheetNum);
				row = sheet.createRow(0);
				// 声明列对象，参数为列索引，可以是0～255之间的任何一个
				// 创建标题,此时row=0,即第一行
				for (int j = 0; j < title.length; j++) {
					cell = row.createCell(j);
					cell.setCellValue(title[j]);
					cell.setCellStyle(xssfCellStyleHeader);
				}
			}
			row = sheet.createRow((i + 1) - (sheetNum * 60000)+sheetNum);
			int cellIndex = 0;
			m = result.get(i);
			for (String str : m) {
				row.createCell((short) cellIndex).setCellValue(str);
				cellIndex++;
			}
		}
	}
	return wb;
}


/**
 * 获取并设置header样式
 */
private static XSSFCellStyle getAndSetXSSFCellStyleHeader(SXSSFWorkbook sxssfWorkbook) {
    XSSFCellStyle xssfCellStyle = (XSSFCellStyle) sxssfWorkbook.createCellStyle();
    Font font = sxssfWorkbook.createFont();
    // 字体大小
    font.setFontHeightInPoints((short) 14);
    // 字体粗细
    //font.setFontName("楷体");
    // 将字体应用到样式上面
    xssfCellStyle.setFont(font);
    // 是否自动换行
    xssfCellStyle.setWrapText(false);
    // 水平居中
    xssfCellStyle.setAlignment(HorizontalAlignment.CENTER);
    // 垂直居中
    xssfCellStyle.setVerticalAlignment(VerticalAlignment.CENTER);
    return xssfCellStyle;
}

}

syfjava

关注

4
点赞
踩
5

收藏

觉得还不错? 一键收藏
0
评论
easyExcel和poi多线程实现百万级数据量导出DEMO

MyCallableTsk 实现 Callable接口。easyExcel导出，controller层接口。NamedThreadFactory 线程池工厂。selectAll() 的具体实现。
复制链接

扫一扫