【ThreadPoolExecutor】多线程实践

分页批量拉取数据

代码片.

    private ThreadPoolExecutor serviceExecutor;

    @PostConstruct
    public void initialization() {
        Integer core = HeraclesServiceUtil.getInt("plan.query.executor.corePoolSize", 3);
        Integer max = HeraclesServiceUtil.getInt("plan.query.executor.maxPoolSize", 15);
        NamedThreadFactory threadFactory = new NamedThreadFactory("planQueryExecutor");

        serviceExecutor = new ThreadPoolExecutor(core, max, 1, TimeUnit.MINUTES, new SynchronousQueue<>(),
                threadFactory, new ThreadPoolExecutor.CallerRunsPolicy());
    }

    public List<TrackUploadReq> getBathObusData(String appId, String appName, Integer totalPage) throws Exception {
        List<TrackUploadReq> trackUploadReqList = new ArrayList<>();

        Long startTime = System.currentTimeMillis();
        Queue<TrackUploadReq> batchAddTrackUploadReqQueue = new ConcurrentLinkedQueue();

        List<Future> futureList = new ArrayList<>();
        for (int i = 1; i < totalPage + 1; i++) {
            int finalI = i;
            AtomicInteger count = new AtomicInteger(0);
            String finalAppName = appName;
            Future subTaskFuture = serviceExecutor.submit(() -> {
                try {
                    ObusResponsBO.Datas dataResults = run(appId, String.valueOf(finalI), count);
                    List<TrackUploadReq> trackUploadReqs = trackObusRecordsConverter
                            .convertObusDataList2TrackUploadReqList(
                            appId, finalAppName, dataResults.getResult());
                    batchAddTrackUploadReqQueue.addAll(trackUploadReqs);
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
            });
            futureList.add(subTaskFuture);
        }
        for (Future future : futureList) {
            future.get();
        }
        trackUploadReqList.addAll(batchAddTrackUploadReqQueue);
        log.debug("总用用时:{}", System.currentTimeMillis() - startTime);
        return trackUploadReqList;
    }

    public ObusResponsBO.Datas run(String appId, String index, AtomicInteger retryCount) throws InterruptedException {
        while (retryCount.get() < maxRetries) {
            try {
                log.debug(Thread.currentThread().getName());
                // 发送HTTP请求并处理响应
                ObusResponsBO.Datas datas = getObusData(appId, index);
                // 处理响应数据，如果响应数据为空，则认为调用失败，需要重试
                if (!datas.getEnabled()) {
                    log.debug("-------调用失败，重试次数：{},index:{}", retryCount, index);
                    //等待retryInterval毫秒后重试
                    Thread.sleep(5);
                    retryCount.getAndIncrement();
                } else {
                    log.debug("调用成功，响应数据：{},index:{},重试次数：{}", datas.toString(), retryCount);
                    // 调用成功，退出循环
                    return datas;
                }
                return datas;
            } catch (Exception e) {
                log.debug("调用失败，重试次数：{},错误原因:{}" + retryCount, e);
                // 等待retryInterval毫秒后重试
                Thread.sleep(5);
                retryCount.getAndIncrement();
            }
        }

        if (retryCount.get() >= maxRetries) {
            log.debug("重试次数达到上限，调用失败。{}", index);
        } else {
            log.debug("调用成功,{}", index);
        }
        return null;
    }