多线程分批导出数据
生产环境有个导出的接口涉及多张表关联查询,sql查询一次300s然后在导出的时候直接超时了,优化完sql之后开始用多线程的方式分批处理数据最后返回一个list生成Excel
我理解的线程只有两种,一种是带返回值的,一直是不带返回值的。
public List<PerInfoExcel> exportPersonnelFiles(PerInfoRequest perInfoRequest) {
List<PerInfoExcel> result = new ArrayList<>();
//request转实体
PerInfo perInfo = PerInfoConvert.initRequest(perInfoRequest);
perInfo.setApprStatus(BusinessConstants.DATA_DON);
if (BusinessConstants.ENT_USER.equals(SecureUtil.getUser().getUserType())) {
//判断不同的查询来源
String entUuid = SecureUtil.getUser().getRegionUuid();
perInfo.setEntUuid(entUuid);
} else {
//判断不同的查询来源
if (StringUtils.isEmpty(perInfoRequest.getDeptCode())) {
perInfo.setDeptCode(SecureUtil.getUserDeptCodeList().get(0));
}
}
if (StringUtil.isNotEmpty(perInfoRequest.getManageAreaName())) {
perInfo.setDeptCode(perInfoRequest.getManageAreaName());
}
long startTime = System.currentTimeMillis();
//导出数据重写
List<PerInfoDTO> list = perInfoMapper.exportInfoPersonnel(perInfo);
long searchTime = (System.currentTimeMillis() - startTime) * 1000;
log.info("sql执行时长为:" + searchTime);
//最终返回数据List
List<PerInfoExcel> resultFinal = new ArrayList<>();
if (!CollectionUtils.isEmpty(list)) {
result = list.stream().map(StaffConvert::convertBOExcel).collect(
Collectors.toList());
//异步,分批处理数据
//线程挂载返回泛型数据
List<Future<List<PerInfoExcel>>> futureList=new ArrayList<>();
// 创建线程池
ExecutorService executor = Executors.newFixedThreadPool(10);
//进一步细化线程挂载数据量
List<List<PerInfoExcel>> partitionList = ListUtil.partition(result, 500);
for ( List<PerInfoExcel> perInfoExcels: partitionList ) {
Future<List<PerInfoExcel>> submit = executor.submit(() -> formatExcelData(allDict,sysRegionVillages,perInfoExcels));
futureList.add(submit);
}
for (Future<List<PerInfoExcel>> future : futureList) {
while (!(future.isDone())) {
log.info("future : " + future.isDone());
}
//如果完成则添加返回值
try {
resultFinal.addAll(future.get());
}catch (Exception e){
e.printStackTrace();
}
}
}
log.info("导出总时间:" + (System.currentTimeMillis() - searchTime) * 1000);
return resultFinal;
}
demo
import cn.hutool.core.lang.UUID;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.*;
/**
* @author dlf
* @date 2023/5/13 16:14
*/
public class TestThread2 {
public static void main(String[] args) throws ExecutionException, InterruptedException {
//最终返回数据
List<String> A=new ArrayList<>();
//带有返回值得线程挂载泛型数据
List<Future<String>> futureList=new ArrayList<>();
// 创建线程池
ExecutorService executor = Executors.newFixedThreadPool(10);
//线程执行数据处理
for (int i = 0; i < 1000; i++) {
Future<String> submit = executor.submit(() -> UUID.randomUUID().toString());
futureList.add(submit);
}
//线程判断数据是否完成
for (Future<String> future : futureList) {
while (!(future.isDone())) {
System.out.println("future : " + future.isDone());
}
//如果完成则添加返回值
A.add(future.get());
}
//处理结束打印结果
System.out.println(A.toString());
System.out.println(A.size());
}
}