手写一个并行处理任务的工具类

 

TaskDisposeUtils是一个并行处理的工具类，可以传入n个任务内部使用线程池进行处理，等待所有任务都处理完成之后，方法才会返回。比如我们发送短信，系统中有1万条短信，我们使用上面的工具，每次取100条并行发送，待100个都处理完毕之后，再取一批按照同样的逻辑发送。 

import org.springframework.util.CollectionUtils;


import java.util.List;

import java.util.concurrent.CountDownLatch;

import java.util.concurrent.ExecutorService;

import java.util.concurrent.Executors;

import java.util.concurrent.TimeUnit;

import java.util.function.Consumer;

import java.util.stream.Collectors;

import java.util.stream.Stream;


/**

 * 微信公众号：javacode2018，获取年薪50万课程

 */

public class TaskDisposeUtils {

    //并行线程数

    public static final int POOL_SIZE;


    static {

        POOL_SIZE = Integer.max(Runtime.getRuntime().availableProcessors(), 5);

    }


    /**

     * 并行处理，并等待结束

     *

     * @param taskList 任务列表

     * @param consumer 消费者

     * @param <T>

     * @throws InterruptedException

     */

    public static <T> void dispose(List<T> taskList, Consumer<T> consumer) throws InterruptedException {

        dispose(true, POOL_SIZE, taskList, consumer);

    }


    /**

     * 并行处理，并等待结束

     *

     * @param moreThread 是否多线程执行

     * @param poolSize   线程池大小

     * @param taskList   任务列表

     * @param consumer   消费者

     * @param <T>

     * @throws InterruptedException

     */

    public static <T> void dispose(boolean moreThread, int poolSize, List<T> taskList, Consumer<T> consumer) throws InterruptedException {

        if (CollectionUtils.isEmpty(taskList)) {

            return;

        }

        if (moreThread && poolSize > 1) {

            poolSize = Math.min(poolSize, taskList.size());

            ExecutorService executorService = null;

            try {

                executorService = Executors.newFixedThreadPool(poolSize);

                CountDownLatch countDownLatch = new CountDownLatch(taskList.size());

                for (T item : taskList) {

                    executorService.execute(() -> {

                        try {

                            consumer.accept(item);

                        } finally {

                            countDownLatch.countDown();

                        }

                    });

                }

                countDownLatch.await();

            } finally {

                if (executorService != null) {

                    executorService.shutdown();

                }

            }

        } else {

            for (T item : taskList) {

                consumer.accept(item);

            }

        }

    }


    public static void main(String[] args) throws InterruptedException {

        //生成1-10的10个数字，放在list中，相当于10个任务

        List<Integer> list = Stream.iterate(1, a -> a + 1).limit(10).collect(Collectors.toList());

        //启动多线程处理list中的数据，每个任务休眠时间为list中的数值

        TaskDisposeUtils.dispose(list, item -> {

            try {

                long startTime = System.currentTimeMillis();

                TimeUnit.SECONDS.sleep(item);

                long endTime = System.currentTimeMillis();


                System.out.println(System.currentTimeMillis() + ",任务" + item + "执行完毕，耗时:" + (endTime - startTime));

            } catch (InterruptedException e) {

                e.printStackTrace();

            }

        });

        //上面所有任务处理完毕完毕之后，程序才能继续

        System.out.println(list + "中的任务都处理完毕!");

    }

}

/**
 * java List 按指定长度分割
 *
 * @param list
 * @param groupSize
 * @return
 */
private List<List<Long>> splitList(List<Long> list, int groupSize) {
    // 使用guava
    return Lists.partition(list, groupSize);
}

 

// 发布的资源历史中 分页12 和 每个资源下挂20 个资料  12*20 个 linkIds，服务使用historyId分表  查询超时 ，所以拆分后循环查询组装
List<List<Long>> splitListLinkIds = splitList(linkIds, 20);
List<TrailCount> trailCountLtList = new ArrayList<>();


StudentQueryExerciseRequest request = new StudentQueryExerciseRequest();
request.setReqId(request_id);
request.setPublishIdList(ids);
request.setExerciseTypeList(Lists.newArrayList(PublishType.ZUOYE.getCode(), PublishType.CEPING.getCode(), PublishType.YUXI.getCode()));
//查询未批改数
// 分线程并行查询
try {
    BlockingQueue<Future<List<TrailCount>>> queue = new LinkedBlockingQueue<Future<List<TrailCount>>>();
    for (List<Long> linkIdsLong : splitListLinkIds) {
        request.setHistoryIdList(linkIdsLong);
        Future<List<TrailCount>> future = executorService.submit(read2List(request));
        queue.add(future);
    }
    int queueSize = queue.size();
    for (int i = 0; i < queueSize; i++) {
        List<TrailCount> list = queue.take().get();
        trailCountLtList.addAll(list);
    }
} catch (Exception e) {
    throw new TeacherPadException(TeacherPadErrorCode.SERVICE_ERR.getValue(), "服务异常");
}