记一次本地TXT或CSV读取入库

记一次本地TXT或CSV读取入库

** ！！！ 当使用分隔符时，注意内容中存在的相同分隔符的问题 **

	public class txt2DB{
    // 一个线程最大处理数据量
    private static final int THREAD_COUNT_SIZE = 5000;
	// 临时存储list最大值
    private static final int LIST_COUNT_SIZE = 50000;
    
    @Resource
    private SmsService smsService;
    
    // 获取
    public List<Sms> getTxtSms() {
        final String path = "G:\\2and4\\20220223.txt";
        FileInputStream inputStream = null;
        Scanner sc = null;
        int count = 0;
        int errCount = 0;
        int queryCount = 0;
        List<Sms> list = new ArrayList<>();
        try {
            inputStream = new FileInputStream(path);
            sc = new Scanner(inputStream, "GBK");
            //一行一行读取数据
            while (sc.hasNextLine()) {
                count ++;
                String line = sc.nextLine();
                if(count < 2) {
                    continue;
                }
                String[] split = line.split(",");
                // 根据分隔符转对象
                Sms sms = new Sms();
                sms.setAccoutId(split[0]);
                sms.setPhoneNumber(split[1]);
                if (count % LIST_COUNT_SIZE == 0) {
                    executeThreadPool(list);
                    list.clear();
                }

                list.add(sms);
            }
            if (sc.ioException() != null) {
                throw sc.ioException();
            }
            // 存储最后读取不够分批最大值的数据
            if (list.size() > 0) {
                executeThreadPool(list);
            }
        } catch (Exception exception) {
            System.out.println(exception.getMessage());
        } finally {
            if (inputStream != null) {
                try {
                    inputStream.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
            if (sc != null) {
                sc.close();
            }
        }
        System.out.println(list.size());
        System.err.println("数据总行：" + count);
        return list;
    }
	//  分批入库
    public void executeThreadPool(List<Sms> smsList) {

        long start = System.currentTimeMillis();

        // 线程数，以5000条数据为一个线程，总数据大小除以5000，再加1
        int round = smsList.size() / THREAD_COUNT_SIZE + 1;
        // 程序计数器
        final CountDownLatch count = new CountDownLatch(round);
        // 创建线程
        ExecutorService executor = Executors.newFixedThreadPool(round);
        // 分配数据
        for (int i = 0; i < round; i++) {
            int startLen = i * THREAD_COUNT_SIZE;
            int endLen = ((i + 1) * THREAD_COUNT_SIZE > smsList.size() ? smsList.size()
                    : (i + 1) * THREAD_COUNT_SIZE);
            final List<Sms> threadList = smsList.subList(startLen, endLen);
            int k = i + 1;
            executor.execute(new Runnable() {
                @Override
                public void run() {
                    // 接口添加数据库,或者进行其他数据操作
                    smsService.saveOrUpdateBatch(threadList);
                    System.out.println("正在处理线程【" + k + "】的数据，数据大小为：" + threadList.size());
                    // 计数器 -1(唤醒阻塞线程)
                    count.countDown();
                }
            });
        }
        try {
            // 阻塞线程(主线程等待所有子线程 一起执行业务)
            count.await();
            long end = System.currentTimeMillis();
            System.out.println(smsList.size() + " 条数据插入查询耗时:" + (end - start) + "ms");
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            // 终止线程池
            // 启动一次顺序关闭，执行以前提交的任务，但不接受新任务。若已经关闭，则调用没有其他作用。
            executor.shutdown();
        }
    }
}