批量插入数据 多线程分批

 分批多线程插入

        int num = 0;
        final ThreadPoolExecutor executor = new ThreadPoolExecutor(4,
                8,
                2, TimeUnit.SECONDS,
                new ArrayBlockingQueue<>(10000), new ThreadPoolExecutor.CallerRunsPolicy());
        executor.allowCoreThreadTimeOut(true);
        int batchSize = 1000;
        int future_size = (availableData.size() + batchSize - 1) / batchSize;
        CountDownLatch countDownLatch = new CountDownLatch(future_size);
        List<Future<Integer>> futures = new ArrayList<>(future_size);
        List<Info> tempList = new ArrayList<>(512);
        for (int i = 0; i < availableData.size(); i++) {
            tempList.add(availableData.get(i));
            //判断批次，够N个数开始处理一波
            if ((i + 1) % batchSize == 0 || (i + 1) == availableData.size()) {
                log.info("处理数据集");
                List<Info> threadList = new ArrayList<>(tempList);
                Future<Integer> future = executor.submit(() -> {
                    int num_t = 0;
                    //todo 你的线程处理，或者数据库操作等
                    for (Info temp : threadList) {
                        try {
                            InfoMapper.addInfo(temp);
                            num_t++;
                        } catch (Exception e) {
                            if (e instanceof DuplicateKeyException) {
                                log.debug("跳过插入重复数据");
                            } else {
                                throw e;
                            }
                        }
                    }
                    threadList.clear();
                    countDownLatch.countDown();
                    return num_t;
                });
                futures.add(future);
                tempList.clear();
            }
        }
        try {
            //设置等待两分钟，两分钟没处理完先返回响应
            countDownLatch.await(2, TimeUnit.MINUTES);
        } catch (InterruptedException e) {
            log.error(e.getMessage(), e);
        }
        for (Future<Integer> future : futures) {
            try {
                num += future.get();
            } catch (InterruptedException | ExecutionException e) {
                log.error(e.getMessage(), e);
            }
        }
        return num;

 简单批量插入

        int num = 0;
        List<List<Info>> list = ListUtils.partition(availableData, 1000);
        SqlSession sqlSession = sqlSessionFactory.openSession(ExecutorType.BATCH, false);
        InfoMapper sqlSession_InfoMapper = sqlSession.getMapper(InfoMapper.class);
        for (List<Info> Infos : list) {
            for (Info availableDatum : Infos) {
                try {
                    sqlSession_InfoMapper.addInfo(availableDatum);
                    num++;
                } catch (Exception e) {
                    if (e instanceof DuplicateKeyException) {
                        log.debug("跳过插入重复数据");
                    } else {
                        throw e;
                    }
                }
            }
            sqlSession.commit();
            sqlSession.clearCache();
        }
        sqlSession.close();
        availableData.clear();
        return num;

两种速度差不多