《代码干货》Quartz集合批量多线程处理

前言

我们经常使用spring quartz 处理各种业务，那么如何使用多线程批量处理，而且保证不会有重复数据。想必处理过这类业务的朋友，一定了解Executors或者ThreadPoolTaskExecutor这类工具。这里不对这些东西做任何解说。因为时间关系，我按自己的方式实现，直接提供代码供大家参考。

需要返值的

public class GameConsumerJob {
    private static final Logger logger = LoggerFactory.getLogger(GameConsumerJob.class);

    @Autowired
    private Service service;

    @Autowired
    private CxylThreadPool threadPool;

    static int threadCounts = 10;//使用的线程数
    static long sum = 0;

    public void execute() {
        logger.info("队列消费者线程运行,GameConsumerJob{}", Thread.currentThread().getName());
        try {
            pushQueueWorkerRun();
        } catch (Exception e) {
            logger.info("队列消费者线程运行,GameConsumerJob{}发生异常：", e);
        }
    }

    private void pushQueueWorkerRun() throws Exception {
        List<CxylConsumerData> dataList = service.getLastConsumers(2000);
        if (dataList != null && dataList.size() > 0) {
            ExecutorService ex = Executors.newFixedThreadPool(threadCounts);
            List<Callable<Long>> callList = new ArrayList<Callable<Long>>();
            int dataCount = dataList.size();
            int len = dataCount/threadCounts;//分批数
            if(len == 0){
                threadCounts = dataCount;//采用一个线程处理List中的一个元素
                len = 1;//重新平均分割List
            }
            for(int i=0; i<threadCounts; i++){
                final List<CxylConsumerData> subList;
                if(i == threadCounts-1){
                    subList = dataList.subList(i*len, dataCount);
                }else{
                    subList = dataList.subList(i*len, len*(i+1)>dataList.size()?dataList.size():len*(i+1));
                }
                //采用匿名内部类实现
                callList.add(new Callable<Long>(){
                    public Long call() throws Exception {
                        long subSum = 0L;
                        for(CxylConsumerData date : subList){
                            int value = cxylZdService.playerConsume(date);
                            subSum += value;
                        }
                        System.out.println("分配给线程："+Thread.currentThread().getName()+"那一部分List的整数和为：\tSubSum:"+subSum);
                        return subSum;
                    }
                });
            }

            List<Future<Long>> futureList = ex.invokeAll(callList);
            for(Future<Long> future: futureList){
                sum += future.get();
            }
            ex.shutdown();
        }
    }

}

使用Callable存在性能消耗，尤其大量子线程创建，以下是无返值优化

    private void pushQueueWorkerRun2() throws Exception {
        List<CxylConsumerData> dataList = service.getLastConsumers(2000);
        if (dataList != null && dataList.size() > 0) {
            ExecutorService exe = Executors.newFixedThreadPool(threadCounts);
            int dataCount = dataList.size();
            int len = dataCount/threadCounts;//分批数
            if(len == 0){
                threadCounts = dataCount;//采用一个线程处理List中的一个元素
                len = 1;//重新平均分割List
            }
            for(int i=0; i<threadCounts; i++){
                final List<CxylConsumerData> subList;
                if(i == threadCounts-1){
                    subList = dataList.subList(i*len, dataCount);
                }else{
                    subList = dataList.subList(i*len, len*(i+1)>dataList.size()?dataList.size():len*(i+1));
                }
                Runnable run = new Runnable() {
                    @Override
                    public void run() {
                        for(CxylConsumerData date : subList){
                            cxylZdService.playerConsume(date);
                        }
                    }
                };
                exe.execute(run);
            }
            exe.shutdown();
            while (true) {
                if (exe.isTerminated()) {
                    logger.info("GameConsumerJob is end {}", Thread.currentThread().getName());
                    break;
                }
                Thread.sleep(200);
            }
            exe.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS);
        }
    }

以上代理仅供大家参考，实现的方式很多，根据自己的业务情况实现即可。