分页查数据当前页提交线程池分批次处理

1.线程池分批次处理抽象类

public abstract class TaskExecutor<T> {

    private static final Logger log = LoggerFactory.getLogger(TaskExecutor.class);

    /**
     * 过滤出需要执行的任务实体列表
     *
     * @param shardId
     * @param startId
     * @param finishId
     * @param limit
     * @return
     */
    public abstract FetchNeedHandleResp<T> fetchNeedHandle(Integer shardId, Long startId, Long finishId, Integer limit);

    /**
     * 转换白名单id执行实体
     *
     * @param shardId
     * @param whiteList
     * @return
     */
    public abstract List<T> convertWhiteToHandle(Integer shardId, List<Long> whiteList);

    /**
     * 执行实体
     *
     * @param task
     */
    public abstract void handleSubTask(T task);

    /**
     * 加锁执行
     *
     * @param lockNamePost
     * @param action
     * @return
     */
    public abstract boolean tryLockAndRun(String lockNamePost, Runnable action);

    /**
     * 实时查询任务执行参数
     *
     * @return
     */
    public abstract TaskExecutorConfig loadConfigInTime();

    /**
     * 多分片执行
     *
     * @param taskContext
     * @param shardItems
     * @param executorService 传入并行执行，不传入串行执行
     */
    public void executeByShards(TaskContext taskContext, List<Integer> shardItems, ExecutorService executorService) {
        if (CollectionUtils.isEmpty(shardItems)) {
            log.error("[executeByShards]分片数据为空，不处理");
            return;
        }
        for (Integer shardItem : shardItems) {
            log.info("[executeByShards] shardId:{}, taskContext:{} start execute", shardItem, GsonUtil.toJson(taskContext.getTaskName()));
            if (Objects.nonNull(executorService)) {
                executorService.submit(() -> execute(adaptShard(taskContext, shardItem)));
                continue;
            }
            execute(adaptShard(taskContext, shardItem));
            log.info("[executeByShards] shardId:{}, taskContext:{} end execute", shardItem, GsonUtil.toJson(taskContext.getTaskName()));
        }
    }

    /**
     * 无分片执行
     *
     * @param taskContext
     */
    public void execute(TaskContext taskContext) {
        TaskInnerContext<T> innerContext;
        try {
            log.info("start>> {}开始于{}", taskContext.getLogBasicInfo(), TimeUtil.formatDateTime(LocalDateTime.now()));
            // 0.参数校验
            validateTaskContext(taskContext);
            // 1.获取任务执行信息
            innerContext = buildWithTaskContext(taskContext);
            // 2.分片过滤
            List<Integer> executeShardList = loadConfigInTime().getExecuteShardList();
            if ((Objects.nonNull(taskContext.getShardId())) && (CollectionUtils.isEmpty(executeShardList) || !executeShardList.contains(taskContext.getShardId()))) {
                log.info("[hitNoShardId]未命中执行分片, 不执行，shardId={}，executeShardList={}", innerContext.getShardId(), executeShardList);
                return;
            }
            // 3.执行任务
            String lockNamePost = Objects.isNull(taskContext.getShardId()) ? innerContext.getTaskName() : innerContext.getTaskName() + "_" + innerContext.getShardId();
            boolean lockRst = tryLockAndRun(lockNamePost, () -> {
                if (CollectionUtils.isNotEmpty(getWhiteList())) {
                    executeParallel(convertWhiteToHandle(innerContext.getShardId(), getWhiteList()), innerContext);
                } else {
                    executeAllData(innerContext);
                }
            });
            AssertUtil.assertTrueIgnorable(lockRst, CommonRespCodeEnum.BIZ_ERROR, "执行任务加锁失败, taskName:{}，shardId={}", innerContext.getTaskName(), innerContext.getShardId());
        } catch (Exception e) {
            ExceptionMonitorUtil.logForException(e, taskContext.getLogBasicInfo());
            return;
        }
        log.info("{}结束于{}, totalSize={}, successSize={}, failureSize={}, failureList={}", taskContext.getLogBasicInfo(), TimeUtil.formatDateTime(LocalDateTime.now()), innerContext.getTotal(), innerContext.getSuccess(), innerContext.getFailure(), innerContext.getFailureList());
    }


    private void executeAllData(TaskInnerContext<T> innerContext) {
        long startId = getStartId();
        int limit = getLimit();
        long finishId = getFinishId();

        while (ifSwitchIsOpen() && startId <= finishId) {
            FetchNeedHandleResp<T> fetchNeedHandleResp = fetchNeedHandle(innerContext.getShardId(), startId, finishId, limit);
            Long nextStartId = fetchNeedHandleResp.getNextStartId();
            List<T> tList = fetchNeedHandleResp.getNeedHandleTasks();
            AssertUtil.assertTrueIgnorable(Objects.nonNull(nextStartId), CommonRespCodeEnum.BIZ_ERROR, "下一个起始ID决策失败，fetchNeedHandleResp={}", GsonUtil.toJson(fetchNeedHandleResp));
            log.info("{}当前批次待执行任务列表大小为{}, 任务={}", innerContext.getLogBasicInfo(), tList.size(), tList);
            if (CollectionUtils.isNotEmpty(tList)) {
                executeParallel(tList, innerContext);
            }
            log.info("{}当前批次id执行完毕, 已完成id totalSize={}, successSize={}, failureSize={}, 任务={}", innerContext.getLogBasicInfo(), innerContext.getTotal(), innerContext.getSuccess(), innerContext.getFailure(), tList);
            startId = fetchNeedHandleResp.getNextStartId();
            tpsLimit(innerContext, tList.size());
        }
    }

    private void executeParallel(List<T> tList, TaskInnerContext<T> innerContext) {
        if (CollectionUtils.isEmpty(tList)) {
            return;
        }
        int asyncCount = getAsyncCount();
        log.info("{}当前并发度为{}", innerContext.getLogBasicInfo(), asyncCount);
        // 获取分组个数
        int partitionSize = tList.size() / asyncCount + (tList.size() % asyncCount == 0 ? 0 : 1);
        List<List<T>> subIdsList = Lists.partition(tList, partitionSize);
        // 根据并发度拆分子任务
        subIdsList.stream()
                .map(subList -> CompletableFuture.runAsync(() -> handle(subList, innerContext), innerContext.getHandleExecutor()))
                .collect(Collectors.toList())
                .forEach(CompletableFuture::join);
    }

    private void handle(List<T> taskList, TaskInnerContext<T> innerContext) {
        taskList.forEach(t -> {
            if (!ifSwitchIsOpen()) {
                log.info("{} handle stop, executeT={}", innerContext.getLogBasicInfo(), GsonUtil.toJson(t));
                return;
            }
            try {
                handleSubTask(t);
                innerContext.getSuccess().incrementAndGet();
                Cat.logEvent("Task.Util", innerContext.getTaskName());
            } catch (Exception e) {
                log.error("{} handle fail, executeT={}", innerContext.getLogBasicInfo(), GsonUtil.toJson(t), e);
                innerContext.getFailure().incrementAndGet();
                innerContext.getFailureList().add(t);
                Cat.logEvent("Task.Util", innerContext.getTaskName(), "fail", GsonUtil.toJson(t));
            }
            innerContext.getTotal().incrementAndGet();
        });
    }

    /**
     * 精确限流,通过当前批次执行的任务量和tps限制计算期望的耗时,根据耗时来决定是否需要sleep来限流
     *
     * @param innerContext 任务上下文
     * @param size         当前批次执行的任务量
     */
    @SneakyThrows
    private void tpsLimit(TaskInnerContext<T> innerContext, long size) {
        // 上一批次任务结束时的时间(毫秒)
        long lastBatchFinishTime = innerContext.getLastBatchFinishTime();
        // tps上限
        List<Integer> executeShardList = loadConfigInTime().getExecuteShardList();
        // 多分片执行时受统一的maxTps限制
        int maxTpsConfig = getMaxTps();
        int maxTps = CollectionUtils.isEmpty(executeShardList) ? maxTpsConfig :
                maxTpsConfig / executeShardList.size() + ((maxTpsConfig % executeShardList.size()) == 0 ? 0 : 1);
        // 期望的当前批次任务结束时间(毫秒)
        long expectedCurrentBatchFinishTime = lastBatchFinishTime + size * 1000L / maxTps;
        // 当前时间
        long currentTimeMillis = System.currentTimeMillis();
        long gapMillis = expectedCurrentBatchFinishTime - currentTimeMillis;
        if (gapMillis > 0L) {
            // 如果期望时间大于系统当前时间,则sleep至期望时间
            Thread.sleep(gapMillis);
        }
        // 记录当前批次任务结束时间
        innerContext.setLastBatchFinishTime(System.currentTimeMillis());
    }

    private void validateTaskContext(TaskContext taskContext) {
        Assert.notNull(taskContext, "任务上下文不能为空");
        Assert.notNull(taskContext.getHandleExecutor(), "任务处理线程池不能为空");
        Assert.hasLength(taskContext.getTaskName(), "任务名不能为空");
    }

    @Data
    public class FetchNeedHandleResp<T> {
        /**
         * 下一个处理的ID
         */
        private Long nextStartId;

        /**
         * 需要执行的任务列表
         */
        private List<T> needHandleTasks;
    }

    @Data
    public class TaskInnerContext<T> extends TaskContext {

        /**
         * 最后一批完成时间
         */
        private long lastBatchFinishTime = System.currentTimeMillis();
        /**
         * 执行总数
         */
        private AtomicInteger total = new AtomicInteger(0);
        /**
         * 执行成功数
         */
        private AtomicInteger success = new AtomicInteger(0);
        /**
         * 执行失败数
         */
        private AtomicInteger failure = new AtomicInteger(0);
        /**
         * 失败列表数
         */
        private List<T> failureList = Collections.synchronizedList(Lists.newArrayList());

    }

    @Data
    public class TaskExecutorConfig {

        /**
         * 任务执行开关是否打开
         */
        private boolean switchIsOpen;

        /**
         * 任务执行白名单
         */
        private List<Long> whiteList;

        /**
         * 任务执行分片列表，不在列表中的分片将不执行
         */
        private List<Integer> executeShardList;

        /**
         * 任务执行异步并发量
         */
        private Integer asyncCount;

        /**
         * 全量执行时每批多少个
         */
        private Integer limit;

        /**
         * 全量执行时TPS上限
         */
        private Integer maxTps;

        /**
         * 执行任务起始Id
         */
        private Long startId;

        /**
         * 执行任务中止Id
         */
        private Long finishId;

    }

    private TaskContext adaptShard(TaskContext taskContext, Integer shardId) {
        TaskContext result = new TaskContext();
        result.setTaskName(taskContext.getTaskName());
        result.setHandleExecutor(taskContext.getHandleExecutor());
        result.setShardId(shardId);
        return result;
    }

    private TaskInnerContext<T> buildWithTaskContext(TaskContext taskContext) {
        TaskInnerContext<T> taskInnerContext = new TaskInnerContext<>();
        if (Objects.nonNull(taskContext)) {
            taskInnerContext.setTaskName(taskContext.getTaskName());
            taskInnerContext.setHandleExecutor(taskContext.getHandleExecutor());
            taskInnerContext.setShardId(taskContext.getShardId());
        }
        return taskInnerContext;
    }

    public boolean ifSwitchIsOpen() {
        return loadConfigInTime().isSwitchIsOpen();
    }

    public List<Long> getWhiteList() {
        TaskExecutorConfig taskExecutorConfig = loadConfigInTime();
        return Objects.isNull(taskExecutorConfig) || CollectionUtils.isEmpty(taskExecutorConfig.getWhiteList()) ? Lists.newArrayList() : taskExecutorConfig.getWhiteList();
    }

    public int getAsyncCount() {
        return defaultIfNotPositive(loadConfigInTime().getAsyncCount(), 1);
    }

    public int getLimit() {
        return defaultIfNotPositive(loadConfigInTime().getLimit(), 50);
    }

    public int getMaxTps() {
        return defaultIfNotPositive(loadConfigInTime().getMaxTps(), 2000);
    }

    public long getStartId() {
        TaskExecutorConfig taskExecutorConfig = loadConfigInTime();
        return Objects.isNull(taskExecutorConfig) || Objects.isNull(taskExecutorConfig.getStartId()) || taskExecutorConfig.getStartId() < 0 ? 0 : taskExecutorConfig.getStartId();
    }

    public long getFinishId() {
        TaskExecutorConfig taskExecutorConfig = loadConfigInTime();
        return Objects.isNull(taskExecutorConfig) || Objects.isNull(taskExecutorConfig.getFinishId()) ? 0 : taskExecutorConfig.getFinishId();
    }

    private int defaultIfNotPositive(Integer value, int defaultValue) {
        return Optional.ofNullable(value)
                .filter(num -> num > 0)
                .orElse(defaultValue);
    }

}

 2.继承该线程池抽象类，分页查询并具体的执行线程操作

@Service
@Slf4j
public class BeautyShopResvTaskExecutor extends TaskExecutor<ShopBookInfoDTO> {
    private static final String TASK_EXECUTOR_CONFIG_LION = "com.sankuai.leads.content.process.task.executor.%s.config";

    /**
     * 丽人后台类目
     */
    private static final Integer LIREN_SHOP_CATEGORY = 2;

    /**
     * 门店状态可预约(否)
     */
    private static final Integer LIREN_SHOP_NON_APPOINTMENT = 0;

    /**
     * 门店状态可预约(是)
     */
    private static final Integer LIREN_SHOP_APPOINTMENT = 1;

    @Autowired
    private PlanQueryRemoteService planQueryRemoteService;

    @Autowired
    private ShopBookInfoRemoteService shopBookInfoRemoteService;

    @Autowired
    private ShopTagMngRemoteService shopTagMngRemoteService;

    @Autowired
    private SinaMtPoiRemoteService sinaMtPoiRemoteService;

    @Override
    public TaskExecutor<ShopBookInfoDTO>.FetchNeedHandleResp<ShopBookInfoDTO> fetchNeedHandle(Integer shardId,
            Long startId, Long finishId, Integer limit) {
        FetchNeedHandleResp<ShopBookInfoDTO> fetchNeedHandleResp = new FetchNeedHandleResp<ShopBookInfoDTO>();
        try {
            PaginatePlanForJobRespDTO respDTO = planQueryRemoteService
                    .paginatePlanForJob(PaginatePlanForJobReqDTOAdapter.adapt(ContentBizTypeEnum.RESERVATION.getCode(),
                            BEAUTY_SHOP_RESV_TAG.getBizCode(), ContentSubjectTypeEnum.MT_SHOP.getCode(),
                            TemplateCodeEnum.NORMAL.getCode(), startId, limit));
            AssertUtils.isTrue(Objects.nonNull(respDTO) && respDTO.isSuccess(), "paginatePlanForJob_RPC调用调用失败");
            List<PlanInfoDTO> planInfoList = respDTO.getPlanInfoList();
            if (CollectionUtils.isEmpty(planInfoList)) {
                fetchNeedHandleResp.setNextStartId(finishId + 1);
                fetchNeedHandleResp.setNeedHandleTasks(Lists.newArrayList());
                return fetchNeedHandleResp;
            }
            long maxPlanId = planInfoList.stream().mapToLong(PlanInfoDTO::getPlanId).max().orElse(finishId);
            fetchNeedHandleResp.setNextStartId(maxPlanId + 1);

            // 获取门店list
            List<Long> mtShopIds = new ArrayList<>();
            for (PlanInfoDTO planInfoDTO : planInfoList) {
                try {
                    // 1. 基础校验
                    AssertUtils.isTrue(Objects.nonNull(planInfoDTO), "任务为空!");

                    // 2. 获取门店
                    Map<String, String> planSubjectMap = planInfoDTO.getPlanSubjectMap();
                    Long mtShopId = MapUtils.isNotEmpty(planSubjectMap)
                            ? NumberUtils.toLong(planSubjectMap.get(SubjectFieldDefEnum.FIELD_1.getKey()), 0L) : null;
                    AssertUtils.checkPositive(mtShopId, "美团门店Id为空!");

                    mtShopIds.add(mtShopId);
                } catch (Exception e) {
                    log.error("[planInfoDTO] planInfoDTO assert error! planInfoDTO:{}", planInfoDTO, e);
                }
            }
            ShopBookInfoQueryByMtShopIdsReqDTO shopBookInfoQueryByMtShopIdsReqDTO = new ShopBookInfoQueryByMtShopIdsReqDTO();
            shopBookInfoQueryByMtShopIdsReqDTO.setMtShopIds(mtShopIds);
            ShopBookInfoQueryByMtShopIdsRespDTO shopBookInfoQueryByMtShopIdsRespDTO = shopBookInfoRemoteService
                    .queryByMtShopIds(shopBookInfoQueryByMtShopIdsReqDTO);
            // 对返回结果进行判断
            if (shopBookInfoQueryByMtShopIdsRespDTO == null
                    || shopBookInfoQueryByMtShopIdsRespDTO.getCommonResp() == null
                    || shopBookInfoQueryByMtShopIdsRespDTO.getCommonResp().getCode() == null
                    || shopBookInfoQueryByMtShopIdsRespDTO.getCommonResp().getCode() != 200) {
                log.error("queryByMtShopIds resp is error,reqDTO:{},respDTO:{}",
                        JSON.toJSONString(shopBookInfoQueryByMtShopIdsReqDTO),
                        JSON.toJSONString(shopBookInfoQueryByMtShopIdsRespDTO));
                throw new RuntimeException("shopBookInfoQueryByMtShopIdsRespDTO is error!!!");
            }
            List<ShopBookInfoDTO> shopBookInfoDTOS = new ArrayList<>(
                    shopBookInfoQueryByMtShopIdsRespDTO.getShopBookInfoDTOMap().values());
            fetchNeedHandleResp.setNeedHandleTasks(shopBookInfoDTOS);
            return fetchNeedHandleResp;
        } catch (Exception e) {
            log.error("[fetchNeedHandle] paginatePlanForJob execute error! startId:{} limit:{} bizCode:{}", startId,
                    limit, BEAUTY_SHOP_RESV_TAG.getBizCode(), e);
            NonIgnorableErrorMonitorUtil.logEvent("BeautyShopResvTaskExecutor_paginatePlanForJobError_fail");
            fetchNeedHandleResp.setNextStartId(startId + limit);
            fetchNeedHandleResp.setNeedHandleTasks(Lists.newArrayList());
        }
        return fetchNeedHandleResp;
    }

    @Override
    public List<ShopBookInfoDTO> convertWhiteToHandle(Integer shardId, List<Long> whiteList) {
        if (CollectionUtils.isNotEmpty(whiteList)) {
            ShopBookInfoQueryByMtShopIdsReqDTO shopBookInfoQueryByMtShopIdsReqDTO = new ShopBookInfoQueryByMtShopIdsReqDTO();
            shopBookInfoQueryByMtShopIdsReqDTO.setMtShopIds(whiteList);

            try {
                ShopBookInfoQueryByMtShopIdsRespDTO shopBookInfoQueryByMtShopIdsRespDTO = shopBookInfoRemoteService
                        .queryByMtShopIds(shopBookInfoQueryByMtShopIdsReqDTO);
                List<ShopBookInfoDTO> shopBookInfoDTOS = shopBookInfoQueryByMtShopIdsRespDTO.getShopBookInfoDTOMap()
                        .values().stream().collect(Collectors.toList());
                return shopBookInfoDTOS;
            } catch (TException e) {
                log.error("[convertWhiteToHandle] convertWhiteToHandle execute error! whiteList: {}", whiteList, e);
            }
        }
        return null;
    }

    @Override
    public void handleSubTask(ShopBookInfoDTO task) {

        // 1. 基础校验
        AssertUtils.isTrue(Objects.nonNull(task), "任务为空!");

        // 2. 获取门店
        Long mtShopId = task.getMtShopId();
        AssertUtils.checkPositive(mtShopId, "美团门店Id为空!");

        // 3. 查询门店类目
        List<Integer> backCategoryTree = sinaMtPoiRemoteService.getBackCategoryTree(mtShopId);
        if (CollectionUtils.isEmpty(backCategoryTree) || !backCategoryTree.contains(LIREN_SHOP_CATEGORY)) {
            return;
        }

        // 5. 打标/去标
        EditorInfoReq editorInfoReq = EditorInfoReqAdapter.adapt();
        if (Integer.valueOf(BookStatusEnum.BOOKABLE.getValue()).equals(task.getStatus())) {
            addTag(mtShopId, TagReqAdapter.adapt(ZdcShopTagEnum.getTagIdByEnv(ZdcShopTagEnum.ONLINE_CAN_BOOK)),
                    editorInfoReq);
        } else if (Integer.valueOf(BookStatusEnum.UN_BOOKABLE.getValue()).equals(task.getStatus())) {
            removeTag(mtShopId, ZdcShopTagEnum.getTagIdByEnv(ZdcShopTagEnum.ONLINE_CAN_BOOK), editorInfoReq);
        }
    }

    @Override
    public boolean tryLockAndRun(String lockNamePost, Runnable action) {
        Lock lock = new ReentrantLock();
        boolean lockAcquired = false;
        try {
            lockAcquired = lock.tryLock();
            if (lockAcquired) {
                log.info("[tryLockAndRun] " + lockNamePost + " 加锁成功,开始执行!");
                action.run();
            }
        } finally {
            if (lockAcquired) {
                lock.unlock();
            }
        }
        return lockAcquired;
    }

    @Override
    public TaskExecutor<ShopBookInfoDTO>.TaskExecutorConfig loadConfigInTime() {
        TaskExecutorConfig config = GsonUtil.fromJson(
                Lion.getString(MdpContextUtils.getAppKey(),
                        String.format(TASK_EXECUTOR_CONFIG_LION, BEAUTY_SHOP_RESV_TAG.getTaskName())),
                TaskExecutorConfig.class);
        AssertUtil.notNull(config, String.format("任务[%s]配置缺失", BEAUTY_SHOP_RESV_TAG.getTaskName()));
        return config;
    }

    private void addTag(long mtShopId, Set<TagReq> tagReqs, EditorInfoReq editorInfoReq) {
        try {
            shopTagMngRemoteService.addTagRelationByMTIdRetry(mtShopId, tagReqs, editorInfoReq);
        } catch (Exception e) {
            log.error("[addTag] addTag error!, mtShopId:{}, tagReqs:{}", mtShopId, GsonUtils.toJson(tagReqs), e);
        }
    }

    private void removeTag(Long mtShopId, Long tagId, EditorInfoReq editorInfoReq) {
        try {
            shopTagMngRemoteService.deleteByMTIdRetry(mtShopId, tagId, editorInfoReq);
        } catch (Exception e) {
            log.error("[removeTag] removeTag error!, mtShopId:{}, tagId:{}", mtShopId, GsonUtils.toJson(tagId), e);
        }
    }
}