分批并行获取数据工具类

背景: 比如当某个接口需要查询的数据量比较大,或者请求耗时比较高,可以进行分批并行获取数据

@Slf4j
@Component
public class BatchHelper {

    /**
     *  获取线程池大小
     */
    private static final int FORK_JOIN_POOL_SIZE = Runtime.getRuntime().availableProcessors() > 4 ? Runtime.getRuntime().availableProcessors() * 2 : 8;

    /**
     * ForkJoinPool 线程池
     */
    private static final ForkJoinPool GENERA_PARALLELDAO_POOL = new ForkJoinPool(FORK_JOIN_POOL_SIZE);
    
    
    /**
     * @param params 集合的元素必须去重，否则多线程查询后聚合的结果可能有重复对象
     */
    public static <T, R> List<R> batchSelect(Collection<T> params, Function<T, List<R>> action) {
        if (params == null || params.isEmpty()) {
            return Collections.emptyList();
        }
        try {
            return GENERA_PARALLELDAO_POOL.submit(() -> params.parallelStream()
                    .flatMap(param -> action.apply(param).stream())
                    .collect(Collectors.toList())).get();
        } catch (Exception e) {
            log.error("batchSelect:" + e.getMessage(), e);
            throw BusinessException.create("batchSelect:" + e.getMessage());
        }
    }

    /**
     * @param splitedSize 分割大小
     * @return
     */
    public static <R, T> List<R> getsParallel(Collection<T> params, Function<Collection<T>, List<R>> action, int maxSize, int splitedSize) {
        if (CollectionUtils.isEmpty(params)) {
            return Collections.emptyList();
        }
        if (params.size() > maxSize) {
            Collection<T> distinctedList = params.stream().distinct().collect(Collectors.toList());
            List<List<T>> splitedList = CollectionSplitUtil.split(distinctedList, splitedSize);
            return batchSelect((List) splitedList, action);
        }
        return action.apply(params);
    }
    
}

业务调用

List<TestDO> = BatchHelper.getsParallel(codeList, codeList-> testDAO.selectByCodes(codeList), 1200, 500);