多线程优化导出

package com.thomas.modules.member.service.impl;

import com.thomas.common.auth.domain.AuthUser;
import com.thomas.common.auth.util.AuthUtil;
import com.thomas.common.file.export.util.UploadUtil;
import com.thomas.common.i18n.model.query.BatchTranslateQuery;
import com.thomas.modules.member.convert.LmsMemberConvert;
import com.thomas.modules.member.enums.MemberIdentityTypeEnum;
import com.thomas.modules.member.enums.MemberSexEnum;
import com.thomas.modules.member.enums.MemberStatusEnum;
import com.thomas.modules.member.model.param.LmsMemberExportParam;
import com.thomas.modules.member.model.query.LmsMemberQuery;
import com.thomas.modules.member.model.vo.LmsMemberExportVO;
import com.thomas.modules.member.model.vo.LmsMemberVO;
import com.thomas.modules.member.repository.mapper.LmsMemberMapper;
import com.thomas.modules.member.service.LmsMemberService;
import com.thomas.modules.role.util.DataPermissionsUtil;
import com.thomas.modules.tencent.client.TencentCosStreamFeignClient;
import com.thomas.modules.translate.client.SysTranslateFeignClient;
import com.thomas.common.concurrent.TraceThreadPoolExecutor;
import com.thomas.common.exception.ServiceException;
import com.thomas.common.model.Result;
import com.thomas.common.util.ThreadUtil;
import com.thomas.excel.util.ExcelUtil;
import com.thomas.feign.multipart.support.ByteArrayMultipartFile;
import com.thomas.i18n.util.I18nUtil;
import io.netty.util.concurrent.DefaultThreadFactory;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.ListUtils;
import org.apache.commons.lang3.StringUtils;
import org.osgi.framework.ServiceException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import javax.annotation.Resource;
import java.io.ByteArrayOutputStream;
import java.util.;
import java.util.concurrent.;
import java.util.stream.Collectors;

import static com.thomas.common.auth.enums.ClientTypeEnum.ADMIN;

/**

• 会员导出service
• @author thomas
• @since 2024-02-25
  */
  @Slf4j
  @Service
  @RequiredArgsConstructor
  public class LmsMemberExportOptimizationService {

@Resource
private LmsMemberMapper lmsMemberMapper;
@Resource
private LmsMemberService lmsMemberService;
@Autowired
private TencentCosStreamFeignClient tencentCosStreamFeignClient;

private final static ThreadPoolExecutor threadPoolExecutor =
new ThreadPoolExecutor(8, 8, 2,
TimeUnit.SECONDS,new LinkedBlockingQueue<>(1000),
Executors. defaultThreadFactory(), new ThreadPoolExecutor.CallerRunsPolicy());

public String exportOptimization(LmsMemberExportParam param) throws Exception {
AuthUser user = AuthUtil.getUser();
LmsMemberQuery memberQuery = LmsMemberConvert.INSTANCE.toQuery(param);
Integer total = lmsMemberMapper.selectCountBimit(memberQuery);

long startTime = System.currentTimeMillis();
List memberExports = new Arraist<>();

if (total < 100000) {
memberExports = this.getLmsMemberExport(user, memberQuery, 0, total);
}else {
memberExports = this.threadExportMember(user, memberQuery, total);
}

long endTime = System.currentTimeMillis();
log.info(“总耗时:{},用户导出总数:{}”,endTime - startTime,memberExports.size());

try {
//上传用户导出文件
ByteArrayOutputStream outputStreamByte = new ByteArrayOutputStream();
ExcelUtil.write(outputStreamByte, memberExports, LmsMemberExportVO.class, I18nUtil::getMessage);
String fileName = “用户导出” + System.currentTimeMillis() + “.xlsx”;
ByteArrayMultipartFile byteArrayMultipartFile = UploadUtil.buildByteArrayMultipartFile(fileName, outputStreamByte.toByteArray());
Result cosResult = tencentCosStreamFeignClient.upload(byteArrayMultipartFile, “xlsx”,null,user.getUserId(),user.getName());

log.info(“用户导出excel地址:{}”,cosResult.requiredData());
return cosResult.requiredData();
} catch (Exception e) {
e.printStackTrace();
throw new ServiceException(“[用户导出]上传文件异常”, e);
}
}

public List threadExportMember(AuthUser user, LmsMemberQuery memberQuery, Integer total) throws Exception {
//获取cpu核数
int coreNum = Runtime.getRuntime().availableProcessors();
coreNum = coreNum < 8 ? 8 : coreNum;
//线程数
int runSize = total % coreNum == 0 ? coreNum : coreNum + 1;
//每个线程处理的数量
int count = total % coreNum == 0 ? total / runSize : ((total - (total % (runSize - 1)) )/ (runSize -1));

// 创建一个线程池，数量和开启线程的数量一样
CountDownLatch countDownLatch = new CountDownLatch(runSize);
List exportVOS = new Arraist<>();

for (int i = 0; i < runSize; i++) {
//第几个线程,0开始
int threadNo = i;
if (i == runSize - 1) {
//最后一个线程导出的数量
count = total % runSize == 0 ? count : total - (count * i);
}

Future<List> future = this.getMemberExportFuture(user, memberQuery, count, countDownLatch, threadNo);
exportVOS.addAll(future.get());
}

countDownLatch.await();
return exportVOS;
}

public Future<List> getMemberExportFuture(AuthUser user, LmsMemberQuery memberQuery, int count,
CountDownLatch countDownLatch, int threadNo) {
return threadPoolExecutor.submit(() -> {
List lmsMemberExport = getLmsMemberExport(user, memberQuery, threadNo, count);
countDownLatch.countDown();
return lmsMemberExport;
});
}

/**
*

• @param user
• @param memberQuery
• @param threadNo 第几个线程
• @param total 每个线程查询总数
• @return
  */
  public List getLmsMemberExport(AuthUser user, LmsMemberQuery memberQuery, int threadNo, int total) {
  List lmsMembers = new Arraist<>(total);
  //每次最多查询5000条
  int size = 5000;
  int offset = threadNo * total;
  size = total > size ? size : total;
  //总的执行查询次数
  int count = total % size == 0 ? total / size : total / size + 1;

for (int i=0;i < count;i++) {
offset = i == 0 ? offset : offset + size;

if (i == count-1) {
//最后一次查询数量
size = total % size == 0 ? size : total % size;
}

List lmsMemberList = lmsMemberMapper.selectPageBimit(offset, size, memberQuery);
lmsMembers.addAll(lmsMemberList);

if (lmsMemberList.size() < size) {
break;
}
}

return this.getMemberExport(lmsMembers,user);
}

public List getMemberExport(List lmsMembers, AuthUser user) {
List<List> partition = ListUtils.partition(lmsMembers, 5000);

List<List> memberExportList = partition.parallelStream().map(members -> {
List memberExports = new Arraist<>();
memberExports.addAll(LmsMemberConvert.INSTANCE.toExportVO(members));

memberExports.forEach(export -> {
export.setAccountStatus(MemberStatusEnum.getByCode(export.getStatus()).getName());
export.setIdentity(MemberIdentityTypeEnum.getByCode(export.getIdentityType()).getName());
int sexEnum = Objects.isNull(export.getSex()) ? MemberSexEnum.SECRET.getCode() : export.getSex();
export.setPersonSex(MemberSexEnum.getByCode(sexEnum).getName());
});

return memberExports;
}).collect(Collectors.toList());

return memberExportList.stream().flatMap(Collection::stream).collect(Collectors.toList());
}

}