多线程分页查询

package com.tomdd.v4;

/**
 * <h1>查询数据的函数接口</h1>
 *
 * @author zx
 * @date 2022年09月01日 11:53
 */
@FunctionalInterface
public interface SelectDataFunction<T, R> {

    R apply(T t);
}

import cn.hutool.core.thread.ThreadFactoryBuilder;
import com.google.common.base.Stopwatch;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.*;

/**
 * <h1>多线分页查询接口</h1>
 *
 * @author zx
 * @date 2022年09月01日 14:07
 */
@Slf4j
@Component
public class MultiThreadGetDataUtils<T> {


    /**
     * 每个线程导出记录最大行数
     */
    private static final int THREAD_MAX_ROW = 20000;
    private static final int THREAD_POOL_SIZE = 10;


    /**
     * <h1>多线程分页查询</h1>
     *
     * @param totalNum       总条数
     * @param timeOut        异步获取超时时间 单位: 秒
     * @param threadMaxRow   每页显示的条数 默认为20000
     * @param threadPoolSize 核心线程数 默认为10
     * @return T 查询的集合类型
     */
    public List<T> multiThreadGetData(long totalNum, String threadPoolName,
                                      SelectDataFunction<Map<String, Object>,
                                              List<T>> selectDataFunction,
                                      Long timeOut, Integer threadMaxRow, Integer threadPoolSize) {
        if (threadMaxRow == null || threadMaxRow == 0) {
            threadMaxRow = THREAD_MAX_ROW;
        }

        if (threadPoolSize == null || threadPoolSize == 0) {
            threadPoolSize = THREAD_POOL_SIZE;
        }

        List<FutureTask<List<T>>> tasks = new ArrayList<>();
        List<T> dataList = new ArrayList<>();

        int loopNum = new Double(Math.ceil((double) totalNum / threadMaxRow)).intValue();
        log.info("多线程查询，总数：{},开启线程数：{}", totalNum, loopNum);
        Stopwatch stopwatch = Stopwatch.createStarted();
        //FIXME 可以自定义线程池，不用JDK提供的线程池
        ExecutorService executorService = Executors
                .newFixedThreadPool(threadPoolSize, new ThreadFactoryBuilder()
                        .setNamePrefix(threadPoolName).build());
        for (int i = 0; i < loopNum; i++) {
            Map<String, Object> map = new HashMap<>(loopNum);
            map.put("offset", i * threadMaxRow);
            if (i == loopNum - 1) {
                map.put("limit", totalNum - threadMaxRow * i);
            } else {
                map.put("limit", threadMaxRow);
            }
            FutureTask<List<T>> task = new FutureTask<>(new GetDataThread<>(map, selectDataFunction));

            tasks.add(task);

            //提交任务到线程池[默认的的无界队列、定义10个线程池]
            // 如果不调用task.get()方法，如果出现异常会被FutureTask内部消化掉的
            executorService.submit(task);
        }

        for (FutureTask<List<T>> task : tasks) {
            try {
                //task.get() 阻塞等待; 没有设置超时时间
                dataList.addAll(task.get(timeOut, TimeUnit.SECONDS));
            } catch (Exception e) {
                e.printStackTrace();
            }
        }

        //关闭线程池
        executorService.shutdown();
        log.info("多线程查询耗时:{}毫秒", stopwatch.stop().elapsed(TimeUnit.MILLISECONDS));
        return dataList;

    }



    public class GetDataThread<V> implements Callable<V> {

        private SelectDataFunction<Map<String, Object>, V> selectDataFunction;

        private Map<String, Object> map;

        public GetDataThread(Map<String, Object> map, SelectDataFunction<Map<String, Object>, V> selectDataFunction) {
            this.map = map;
            this.selectDataFunction = selectDataFunction;

        }


        @Override
        public V call() throws Exception {
            log.info("线程名称:{},查询条件:[offset:{}/limit:{}]", Thread.currentThread().getName(), map.get("offset"), map.get("limit"));
            return selectDataFunction.apply(map);
        }
    }

}

需要优化的地方:

1. 使用自定义的线程池(包括拒绝策略)
   2.如果出现异常如何处理

写这个工具类是为类工作上有一个异步导出服务。后面做完了再分享。