Java多线程分批导入数据

package com.rk.iam.sys.service;

/**
 * @author wdy
 * @version 1.0
 * @date 2022/4/27 14:50
 */

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.serializer.SerializerFeature;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.rk.iam.sys.common.vo.SysOrgVO;
import com.rk.iam.sys.entity.SysOrg;
import com.rk.unified.common.utils.JwtRequestUtils;
import com.rk.unified.common.vo.Cuser;
import com.sun.org.slf4j.internal.Logger;
import com.sun.org.slf4j.internal.LoggerFactory;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

/**
 * 组织导入线程类
 * @author wdy
 *
 */
@Service
public class OperationOrgsService {

    private static final Logger log = LoggerFactory.getLogger(OperationOrgsService.class);
    //线程计数器
    private CountDownLatch threadsSignal;
    //每个线程处理的数据量
    private static final int count=1000;
    @Autowired
    private SysOrgService sysOrgService;
    @Autowired
    private JwtRequestUtils jwtRequestUtils;
    //定义线程池数量为8,每个线程处理1000条数据
    private static ExecutorService execPool = Executors.newFixedThreadPool(8);

    /**
     * 多线程批量执行插入
     * @return
     */
    public String batchAddData(List<SysOrgVO> list) throws InterruptedException, Exception {
           Map<Integer,String> response=new HashMap<>();
        if(list.size()<=count) {
            threadsSignal=new CountDownLatch(1);
            execPool.submit(new InsertDate(list));
        }else {
            List<List<SysOrgVO>> li=createList(list, count);
            threadsSignal=new CountDownLatch(li.size());
            for(List<SysOrgVO> liop:li) {
                execPool.submit(new InsertDate(liop));
            }
        }
        response.put(0,"success");
        threadsSignal.await();
        return JSON.toJSONString(response, SerializerFeature.WriteMapNullValue);
    }

    /**
     * 数据拆分
     * @param targe
     * @param size
     * @return
     */
    public static List<List<SysOrgVO>>  createList(List<SysOrgVO> targe,int size) {
        List<List<SysOrgVO>> listArr = new ArrayList<List<SysOrgVO>>();
        //获取被拆分的数组个数
        int arrSize = targe.size()%size==0?targe.size()/size:targe.size()/size+1;
        for(int i=0;i<arrSize;i++) {
            List<SysOrgVO>  sub = new ArrayList<SysOrgVO>();
            //把指定索引数据放入到list中
            for(int j=i*size;j<=size*(i+1)-1;j++) {
                if(j<=targe.size()-1) {
                    sub.add(targe.get(j));
                }
            }
            listArr.add(sub);
        }
        return listArr;
    }


    /**
     * 内部类,开启线程批量保存数据
     * @author wdy
     *
     */
    class  InsertDate  extends Thread{
        List<SysOrgVO> lientity=new ArrayList<SysOrgVO>();
        Cuser cuser = jwtRequestUtils.getCuser();
        public  InsertDate(List<SysOrgVO> limodel){
            limodel.forEach((model)->{
                lientity.add(model);
            });
        }
        @Override
        public void run() {
            create(lientity, cuser);
            threadsSignal.countDown();
        }
    }

    /*
    *
    * 保存数据到数据库
    * */
    public void create(List<SysOrgVO> sysOrgVOList, Cuser cuser) {
        sysOrgVOList.forEach(sysOrgVO -> {
            String name = sysOrgVO.getName();
            Long id = 0L;
            List<SysOrg> sysOrgList = new ArrayList<>();
            boolean flag = false;
            if (name.contains("/")) {
                String[] split = name.split("/");
                if (split.length > 0) {
                    for (int i = 0; i < split.length; i++) {
                        sysOrgVO.setCid(cuser.getCid());
                        sysOrgVO.setRemark("批量导入组织");
                        if (i == 0) {
                            List<SysOrg> list = sysOrgService.lambdaQuery().eq(SysOrg::getCid, cuser.getCid()).eq(SysOrg::getName, split[i]).list();
                            if (CollectionUtils.isEmpty(list)) {
                                sysOrgVO.setParentId(0L);
                                sysOrgVO.setName(split[i]);
                                SysOrg sysOrg = sysOrgService.convertToEntity(sysOrgVO);
                                sysOrgService.save(sysOrg);
                                id = sysOrg.getId();
                            } else {
                                /*
                                 * 获取根目录下父级目录
                                 * */
                                for (SysOrg sysOrg : list) {
                                    sysOrgList = sysOrgService.getBaseMapper()
                                            .selectList(new LambdaQueryWrapper<SysOrg>()
                                                    .eq(SysOrg::getCid, cuser.getCid())
                                                    .eq(SysOrg::getParentId, sysOrg.getId()));

                                }
                                id = list.get(0).getId();
                            }
                        } else {
                            if (CollectionUtils.isNotEmpty(sysOrgList)) {
                                for (SysOrg sysOrg : sysOrgList) {
                                    if (split[i].equals(sysOrg.getName())) {
                                        flag = true;
                                        break;
                                    }
                                }

                            }

                            if (!flag) {
                                sysOrgVO.setParentId(id);
                                sysOrgVO.setName(split[i]);
                                SysOrg sysOrg = sysOrgService.convertToEntity(sysOrgVO);
                                sysOrgService.save(sysOrg);
                                id = sysOrg.getId();
                            }
                        }
                    }
                }
            }
        });
    }
}