消息队列，多线程导入csv文件

大体思路：消息队列，单线程获取数据放入队列，多线程消费队列

话不多说，看代码

CsvJobService

CsvJobService抽象类，单线程生产者，多线程消费者

package org.meichao.demo.service.csvJob;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;

import java.io.BufferedReader;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.*;

@Service
public abstract class CsvJobService {

    private static final Logger LOGGER = LoggerFactory.getLogger(CsvJobService.class);

    /**
     * deque的大小
     */
    public static final int DEQUE_SIZE = 10000;

    /**
     * 线程数量
     */
    public static final int THREAD_SIZE = 16;

    /**
     * 数据单次入库条数
     */
    public static final int SINGLE_SIZE = 1000;

    /**
     * 导入cvs文件
     * @param filePath 文件路径
     * @param charSet 文件编码
     */
    public void importCVSFile(String filePath,String charSet){
        //阻塞队列,采用独立的写锁putLock和读锁takeLock,高并发读写行性能比ArrayBlockingQueue好
        BlockingDeque deque = new LinkedBlockingDeque(DEQUE_SIZE);
        //开始时间
        long startTime = System.currentTimeMillis();

        //生产者 读取cvs文件，数据放入deque
        ExecutorService product = Executors.newSingleThreadExecutor();
        product.execute(() -> {
            try {
                BufferedReader reader = Files.newBufferedReader(Paths.get(filePath, charSet));
                for (int i = 0; ; i++) {
                    String line = reader.readLine();
                    if (line == null) {
                        LOGGER.info("------生产者完成：总数据：{} 条", i);
                        break;
                    }
                    deque.put(line);
                }
            } catch (Exception e) {
                LOGGER.info("------读取文件异常 e:{}", e);
            }
        });

        CountDownLatch countDownLatch = new CountDownLatch(THREAD_SIZE);
        //固定大小的子线程池
        ExecutorService executorService = Executors.newFixedThreadPool(THREAD_SIZE);
        //防止阻塞，多线程哪个执行完先返回哪个
        ExecutorCompletionService<Integer> executorCompletionService = new ExecutorCompletionService(executorService);
        for (int i = 0; i < THREAD_SIZE; i++) {
            //循环开启线消费者线程
            executorCompletionService.submit(new Customer(deque,countDownLatch));
        }

        try {
            //等待子线程执行完
            countDownLatch.await();
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        int jobSum = 0;
        for (int i = 0; i < THREAD_SIZE; i++) {
            try {
                //子线程结果
                Integer temp = executorCompletionService.take().get();
                jobSum += temp;
            } catch (InterruptedException e) {
                e.printStackTrace();
            } catch (ExecutionException e) {
                e.printStackTrace();
            }
        }

        product.shutdown();
        executorService.shutdown();
        long endTime = System.currentTimeMillis();
        LOGGER.info("导入任务完成,总数据：{}条,用时：{}秒", jobSum, (endTime - startTime) / 1000);
    }

    /**
     * 消费者
     */
    class Customer implements Callable{

        private BlockingDeque<String> deque;
        private CountDownLatch countDownLatch;

        public Customer(BlockingDeque deque, CountDownLatch countDownLatch) {
            this.deque = deque;
            this.countDownLatch = countDownLatch;
        }

        @Override
        public Integer call() throws Exception {
            int sum = 0;
            List<String> list = new LinkedList();
            //循环消费队列中的数据
            for (;;){
                String poll = deque.poll(1, TimeUnit.SECONDS);
                if(poll == null){
                    sum += list.size();
                    saveJob(list);
                    list.clear();
                    break;
                }
                list.add(poll);
                //当list大小>=设定的单次数据量，执行saveJob导入
                if (list.size() >= SINGLE_SIZE) {
                    sum += list.size();
                    saveJob(list);
                    list.clear();
                }
            }
            countDownLatch.countDown();
            return sum;
        }
    }

    /**
     * 数据处理方法
     * @param list
     */
    public abstract void saveJob(List<String> list);

}

CsvDemoService

CsvDemoService继承CsvJobService，实现具体的saveJob方法

package org.meichao.demo.service.csvJob;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import java.util.List;

@Service
public class CsvDemoService extends CsvJobService {

    private static final Logger LOGGER = LoggerFactory.getLogger(CsvDemoService.class);

    @Override
    public void saveJob(List<String> list) {
        if (list == null || list.size() <= 0) {
            return;
        }

        LOGGER.info("------注入Mapper，执行insertSql------");
    }
}

CsvController

CsvController注入CsvDemoService或者注入工厂类CsvJobServiceFactory，调用importCVSFile方法（此方法由继承而来）

package org.meichao.demo.service.csvJob;

import org.meichao.utils.RespInfoUtils;
import org.meichao.utils.vo.RespInfo;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;

@RestController
@RequestMapping("/csv")
public class CsvController {
    //service多可使用工厂类
    @Autowired
    private CsvJobServiceFactory csvJobServiceFactory;

    @RequestMapping(value = "/importCsvDataDemo" ,method = RequestMethod.GET)
    public RespInfo importCsvDataDemo(@RequestParam("filePath")String filePath,
                                  @RequestParam("charSet")String charSet){
        csvJobServiceFactory.getInstance("demo").importCVSFile(filePath, charSet);
        return RespInfoUtils.create().respSuccess("成功导入数据").toRespInfo();
    }
}

CsvJobServiceFactory

如果service比较多，可以使用工厂模式

package org.meichao.demo.service.csvJob;

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

/**
 * 如果service比较多，可以使用工厂模式
 */
@Component
public class CsvJobServiceFactory {

    @Autowired
    private CsvDemoService csvDemoService;

    public CsvJobService getInstance(String name){
        if("demo".equals(name))
            return csvDemoService;
        return null;
    }
}

完结撒花