java导入大数据（多线程导入）

 1、Controller

public AjaxResult importData(MultipartFile file) throws Exception {

        if (!file.isEmpty()) {
            //
            String filename = file.getOriginalFilename();
            String prefix = filename.substring(filename.lastIndexOf(".") + 1);
            // Excel 2007版9
            if (null != prefix && prefix.equalsIgnoreCase("xlsx")) {
                // 返回记录数
                String msg = "";
                try {
                    File f = null;
                    f = File.createTempFile("tmp", null);
                    file.transferTo(f);
                    f.deleteOnExit();
                    // 文件路径
                    msg = objectService.parseAndinsertDb(f.getAbsolutePath());
                } catch (IOException e) {
                    e.printStackTrace();
                }
                return AjaxResult.success(msg);
            } else {
                return AjaxResult.error( "请选择文件");
            }


        } else {
            return AjaxResult.error( "文件不存在");
        }
    }

2、Service

@Override
    public String parseAndinsertDb(String filepath) throws Exception {

        final int maxThreadNums = 5;
        final Boolean[] endFlag = {false};
        final List<Integer> num = new ArrayList<>();
        long startMini = System.currentTimeMillis();
        final BlockingQueue<<自己对应实体> results = new LinkedBlockingQueue<自己对应实体>();
        // 多线程插入
        final CyclicBarrier barrier = new CyclicBarrier(maxThreadNums + 1);
        ExecutorService executorService = Executors.newFixedThreadPool(maxThreadNums);
        executorService.execute(new Runnable() {
            @Override
            public void run() {        // 解析
                ParseXMLUtil xmlUtil = new ParseXMLUtil(results, <自己对应实体.class);
                try {
                    xmlUtil.processFirstSheet(filepath);
                    endFlag[0] = true;
                    barrier.await();
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
        });
        Thread.sleep(1000);
        for (int i = 1; i < maxThreadNums; i++) {
            // 提交任务
            executorService.execute(new InsertDataTask<自己对应实体>(results, barrier, "task" + i, endFlag, num, 自己对应实体Mapper));
        }
        barrier.await();
        executorService.shutdown();

        long endMini = System.currentTimeMillis();
        Integer a = 0;
        for (Integer n : num) {
            a += n;
        }
        return "处理数据：" + a + " 条，用时：" + (endMini - startMini) / 1000.0 + "s";
    }

 

3、InsertDataTask

BaseMapper为本人tk.mybatis mapper可以引用自己的mapper

import com.ruoyi.common.core.dao.BaseMapper;
import lombok.SneakyThrows;
import org.springframework.beans.factory.annotation.Autowired;

import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.BrokenBarrierException;
import java.util.concurrent.CyclicBarrier;
import java.util.concurrent.locks.ReentrantLock;


public class InsertDataTask<T> implements Runnable {

    @Autowired
    private final BaseMapper baseMapper;

    private final BlockingQueue<T> queue;

    private final CyclicBarrier barrier;

    private final String taskName;

    private final Boolean[] endFlag;
    private final List<Integer> num;

    public InsertDataTask(BlockingQueue<T> queue, CyclicBarrier barrier, String taskName, Boolean[] endFlag, List<Integer> num, BaseMapper baseMapper) {
        this.queue = queue;
        this.barrier = barrier;
        this.taskName = taskName;
        this.endFlag = endFlag;
        this.num = num;
        this.baseMapper = baseMapper;
    }

    @SneakyThrows
    @Override
    public void run() {
        System.out.println(taskName + "正在执行");

        List<T> list = new ArrayList<T>(1000);
        final ReentrantLock lock = new ReentrantLock();
        final int[] a = {0, 0, 0, 0};
        while (endFlag[0] == false) {
            a[0]++;
            if (queue.isEmpty()) {
                Thread.sleep(500);
                continue;
            }
            while (!queue.isEmpty()) {
                lock.lock();
                list.clear();
                a[1]++;
                if (endFlag[0] == false && queue.size() < 10000) {
                    Thread.sleep(100);
                    continue;
                }
                // 出栈
                queue.drainTo(list, 20000);

                //System.out.println(taskName + "当前执行数量list " + list.size() + " queue " +queue.size());
                //分批处理
                if (null != list && list.size() > 0) {
                    a[2]++;

                    //System.err.println(taskName + " 插入size +++> " + list.size());
                    int pointsDataLimit = 20000;//限制条数
                    Integer size = list.size();
                    a[3] += size;

                    num.add(size);//统计数量
                    //判断是否有必要分批
                    if (pointsDataLimit < size) {
                        int part = size / pointsDataLimit;//分批数
                        // System.out.println("共有 ： "+size+"条，！"+" 分为 ："+part+"批");
                        for (int i = 0; i < part; i++) {
                            //1000条
                            List<T> listPage = list.subList(0, pointsDataLimit);
                            baseMapper.insertList(listPage);
                            //剔除
                            list.subList(0, pointsDataLimit).clear();
                        }
                        if (!list.isEmpty()) {
                            //表示最后剩下的数据
                            baseMapper.insertList(list);
                        }
                    } else {
                        baseMapper.insertList(list);
                    }
                } // end if

                if (queue.isEmpty()) {
                    try {
                        Thread.sleep(1000);
                    } catch (InterruptedException e) {
                        e.printStackTrace();
                    }
                }
                lock.unlock();
            }
        } // while
        System.out.println(taskName + "执行完毕，等待其他结果" + "，外循环次数1==》 " + a[0] + "，内循环次数2==》 " + a[1] + "，内循环执行次数2==》 " + a[2] + "，内循环执行list数==》 " + a[3]);
        try {
            barrier.await();
        } catch (InterruptedException e) {
            e.printStackTrace();
        } catch (BrokenBarrierException e) {
            e.printStackTrace();
        }
    }
}