工作纪实45-记一段多线程批处理的代码

数据很多，多线程分批处理

@Slf4j
public class ThreadPoolUtils {
	public static <R> List<R> batchExecute(String taskName, List<?> originList, Function<List<?>, Callable<R>> supplier) {
        // 多线程批处理
        int batchSize = ContentFeedpostConfig.batchZPCreativeLimit;
        List<? extends List<?>> partition = Lists.partition(originList, batchSize);
        CompletionService<R> completionService = new ExecutorCompletionService<>(ThreadPoolUtils.getPreparePoolExecutor());
        List<Future<R>> futureList = Lists.newArrayListWithCapacity(4);
        for (List<?> list : partition) {
            Callable<R> apply = supplier.apply(list);
            futureList.add(completionService.submit(apply));
        }
        List<R> result = Lists.newArrayListWithCapacity(4);
        for (int n = 0; n < futureList.size(); n++) {
            try {
                Future<R> future = completionService.poll(10, TimeUnit.MILLISECONDS);
                if (future != null) {
                    R r = future.get();
                    if (r != null) {
                        result.add(r);
                    }
                }
            } catch (TimeoutException overtime) {
                MetricUtils.EXCEPTION_COUNTER.labels(taskName).inc();
                if (EXCEPTION_LOG_RATE_LIMITER.tryAcquire()) {
                    log.error("{} timeout exception ", taskName, overtime);
                }
            } catch (Exception e) {
                if (e instanceof InterruptedException) {
                    Thread.currentThread().interrupt();
                }
                MetricUtils.EXCEPTION_COUNTER.labels(taskName).inc();
                if (EXCEPTION_LOG_RATE_LIMITER.tryAcquire()) {
                    log.error("{} unknown exception ", taskName, e);
                }
            }
        }
        return result;
    }
}

public static Table<Long, String, Object> queryZhaoPinExplain(Map<String, String> firstRequest, List<Info> infoList) {
        // 参数预处理
        DispLocal dispLocal = ParamsUtils.getZhaoPinUserLocal(firstRequest);
        DispCategory dispCategory = ParamsUtils.getZhaoPinCate();
        // 多线程批处理
        List<ExplanationResult> batchExecuteResult = ThreadPoolUtils.batchExecute("queryZhaoPinExplain", infoList, list -> {
            List<ExplanationJob> jobList = Lists.newArrayListWithCapacity(list.size());
            list.forEach(info -> {
                ExplanationJob job = new ExplanationJob();
                job.setInfo((Info) info);
                jobList.add(job);
            });
            ParamInfo paramInfo = new ParamInfo();
            paramInfo.setSlot("appindex_joblist");// 首页是固定的，如果有其他场景需要适配，需要通知招聘业务线进行开发
            paramInfo.setSid(firstRequest.getOrDefault("fid", ""));
            paramInfo.setUserid(firstRequest.getOrDefault("userid", ""));
            paramInfo.setPlatform(3);// app
            paramInfo.setIp(firstRequest.getOrDefault("clientIp", ""));// 客户端id
            paramInfo.setImei(firstRequest.getOrDefault("imei", ""));
            paramInfo.setParams("{\"personPri\":\"P1\"}");
            paramInfo.setDisplocal(dispLocal);
            paramInfo.setDispcate(dispCategory);
            return () -> explainService.getExplanationResult(jobList, paramInfo);
        });
        // 返回值处理
        Table<Long, String, Object> result = HashBasedTable.create();
        for (ExplanationResult scfReturn : batchExecuteResult) {
            List<ExplanationInfo> explanationResult = scfReturn.getExplanationInfos();
            // 获取标题、标签、推荐理由
            explanationResult.forEach(k -> {
                long infoID = k.getInfoID();
                ExplanationSchema title = k.getExplanationSchemaFromDomain(ExplanationInfoDomainEnum.TITLE);
                if (!title.isWithMultipleValues()) {
                    result.put(infoID, "title", title.getText());
                }
                ExplanationSchema label = k.getExplanationSchemaFromDomain(ExplanationInfoDomainEnum.LABEL);
                if (label.isWithMultipleValues()) {
                    List<String> list = label.getValues().stream().map(ExplanationSchema::getText).collect(Collectors.toList());
                    result.put(infoID, "tags", list);
                }
                ExplanationSchema reason = k.getExplanationSchemaFromDomain(ExplanationInfoDomainEnum.REASON);
                if (reason.isWithMultipleValues()) {
                    List<String> list = reason.getValues().stream().map(ExplanationSchema::getText).collect(Collectors.toList());
                    result.put(infoID, "reasons", list);
                }
            });
        }
        return result;
    }

CompletionService是一个接口，它继承自Executor接口，并为提交的任务提供了一种获取完成结果的机制。在使用CompletionService时，常用的方法包括：

submit(Callable task)：将任务提交给CompletionService进行处理，并返回表示任务结果的Future对象。
submit(Runnable task, T result)：将任务提交给CompletionService进行处理，并返回表示任务结果的Future对象，此处的result参数用于任务执行完成后返回的结果。
take()：获取已完成的任务的结果，并返回表示任务结果的Future对象。如果没有已完成的任务，take()方法会阻塞等待任务完成。
poll()：尝试立即获取已完成的任务的结果，并返回表示任务结果的Future对象。如果没有已完成的任务，poll()方法会立即返回null。
poll(long timeout, TimeUnit unit)：在指定的时间内，尝试获取已完成的任务的结果，并返回表示任务结果的Future对象。如果在指定时间内没有已完成的任务，poll()方法会返回null。
这些方法提供了不同的方式来获取已完成任务的结果，take()方法会阻塞等待任务完成，而poll()方法则是立即返回结果或者null。

变种代码，如果定义和结果获取需要分开的话

	public static <R> List<Future<R>> batchExecuteSubmit(int batchSize, List<?> originList, Function<List<?>, Callable<R>> supplier, CompletionService<R> completionService) {
        // 多线程批处理
        List<? extends List<?>> partition = Lists.partition(originList, batchSize);
        List<Future<R>> futureList = Lists.newArrayListWithCapacity(4);
        for (List<?> list : partition) {
            Callable<R> apply = supplier.apply(list);
            futureList.add(completionService.submit(apply));
        }
        return futureList;
    }

    public static <R> List<R> batchExecutePost(String taskName, List<Future<R>> futureList, CompletionService<R> completionService){
        List<R> result = Lists.newArrayListWithCapacity(4);
        for (int n = 0; n < futureList.size(); n++) {
            try {
                Future<R> future = completionService.poll(timeout, TimeUnit.MILLISECONDS);
                if (future != null) {
                    R r = future.get();
                    if (r != null) {
                        result.add(r);
                    }
                }
            } catch (Exception e) {
                if (e instanceof InterruptedException) {
                    Thread.currentThread().interrupt();
                }
                MetricUtils.EXCEPTION_COUNTER.labels(taskName).inc();
                if (EXCEPTION_LOG_RATE_LIMITER.tryAcquire()) {
                    log.error("{} unknown exception ", taskName, e);
                }
            }
        }
        return result;
    }