多线程查库入库业务实现。

多线程查库入库业务实现。

以下两种写法起到的效果相同

1、
线程池 + semaphore 限流

@Component
@Slf4j
public class LowQualityUserAnalyseFtpFileJob {


    @Autowired
    ServiceZcdmbmStatisticDao serviceZcdmbmStatisticDao;
    @Autowired
    LqTyUserQualityInfoDao lqTyUserQualityInfoDao;

    private Semaphore semaphore = new Semaphore(5);
    //线程池不要每次都去new,拿出来外面，入库线程池
    private ThreadPoolExecutor executor =
            new ThreadPoolExecutor(5, 5, 2L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(1000), new ThreadPoolExecutor.DiscardOldestPolicy());




    public ReturnT<String> lowQualityUserDetailHandler(String param) {
        XxlJobLogger.log("开始执行定时任务[lowQualityUserDetailHandler]……");
        try {
            Integer count = lqTyUserQualityInfoDao.getCount();
            int num = 3000;//分批查询，每次查询插入的数量
            int times = (int) Math.ceil(count * 1.0 / num);//循环查询多少次
            //全量同步前，先清空表
            serviceZcdmbmStatisticDao.truncate();
            //查库，入库
            for (int i = 0; i < times; i++) {
                final int start = i * num + 1;
                final int end = (i + 1) * num;
                // 创建一个 Runnable 对象，并提交到线程池中执行
                executor.submit(() -> {
                    try {
                        // 获取一个信号量许可证，表示当前线程占用了一个数据库连接
                        semaphore.acquire();
                        // 批量查库，入库
                        List<ServiceZcdmbmStatistic> serviceZcdmbmStatisticList = lqTyUserQualityInfoDao.getServiceZcdmbmStatisticLst(start, end);
                        serviceZcdmbmStatisticDao.insertBat(serviceZcdmbmStatisticList);
                    } catch (InterruptedException e) {
                        e.printStackTrace();
                    } finally {
                        // 释放一个信号量许可证，使得其他线程可以获取许可证并执行任务
                        semaphore.release();
                    }
                });
            }
        } catch (Exception e) {
            XxlJobLogger.log("定时任务异常");
            XxlJobLogger.log(e);
            log.info("定时任务异常");
        }
        return ReturnT.SUCCESS;
    }
   


}

2、
线程池 + semaphore 限流，
但把查库，入库封装成一个task任务，方便semaphore控制

@Component
@Slf4j
public class LowQualityUserAnalyseFtpFileJob {


    @Autowired
    ServiceZcdmbmStatisticDao serviceZcdmbmStatisticDao;
    @Autowired
    LqTyUserQualityInfoDao lqTyUserQualityInfoDao;

    private Semaphore semaphore = new Semaphore(5);
    //线程池不要每次都去new,拿出来外面，入库线程池
    private ThreadPoolExecutor executor =
            new ThreadPoolExecutor(5, 5, 2L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(1000), new ThreadPoolExecutor.DiscardOldestPolicy());


    @XxlJob("lowQualityUserDetailHandler")
    public ReturnT<String> lowQualityUserDetailHandler(String param) {
        XxlJobLogger.log("开始执行定时任务[lowQualityUserDetailHandler]……");
        try {
            Integer count = lqTyUserQualityInfoDao.getCount();
            int num = 3000;//分批查询，每次查询插入的数量
            int times = (int) Math.ceil(count * 1.0 / num);//循环查询多少次
            //全量同步前，先清空表
            serviceZcdmbmStatisticDao.truncate();
            //查库，入库
            for (int i = 0; i < times; i++) {
                semaphore.acquire();
                executor.execute(new ProcessBatchTask(i * num + 1, (i + 1) * num));
            }
        } catch (Exception e) {
            XxlJobLogger.log("定时任务异常");
            XxlJobLogger.log(e);
            log.info("定时任务异常");
        }
        return ReturnT.SUCCESS;
    }


    //把查库，入库封装成一个task任务，方便semaphore控制，做到线程安全
    private class ProcessBatchTask implements Runnable {

        private int startIndex;
        private int endIndex;

        public ProcessBatchTask(int startIndex, int endIndex) {
            this.startIndex = startIndex;
            this.endIndex = endIndex;
        }

        @Override
        public void run() {
            //多线程查库，入库
            try {
                List<ServiceZcdmbmStatistic> serviceZcdmbmStatisticList = lqTyUserQualityInfoDao.getServiceZcdmbmStatisticLst(startIndex, endIndex);
                serviceZcdmbmStatisticDao.insertBat(serviceZcdmbmStatisticList);
            }
            finally {
                semaphore.release();
            }
        }

    }

}