需要的依赖 hutool-core,lombok,分页异步执行,生产者为分页主线程执行,消费者为异步线程池执行
import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.date.TimeInterval;
import cn.hutool.json.JSONUtil;
import lombok.extern.slf4j.Slf4j;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.*;
import java.util.function.Consumer;
import java.util.function.Function;
@Slf4j
public class PageTaskUtil {
private final static ExecutorService executorService = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() * 2+1);
/**
* 分页异步执行,生产者为分页主线程执行,消费者为异步线程池执行
* @param count 生产者记录总数
* @param pageSize 分页大小
* @param getList 获取生产者数据,入参为 分页开始索引
* @param callableFunction 消费数据
* @return 执行任务的返回结果用,分割
*/
public static <T, U> String exe(Long count, Integer pageSize, Function<Integer,List<T>> getList, Function<T,Callable<U>> callableFunction){
ArrayList<U> failRecords = new ArrayList<>();
TimeInterval timer = DateUtil.timer();
List<Future<ArrayList<U>>> futureListProducer = new ArrayList<>();
for (int i = 0; (long) i * pageSize < count; i++) {
int finalI = i;
Callable<ArrayList<U>> producer = () -> {
ArrayList<U> failRecord = new ArrayList<>();
try {
//获取列表数据
List<T> empNos = getList.apply(pageSize * finalI);
List<Future<U>> futureList = new ArrayList<>();
empNos.forEach(x -> {
//处理数据
FutureTask<U> task = new FutureTask<>(callableFunction.apply(x));
futureList.add(task);
executorService.execute(task);
});
//获取处理数据的返回值
for (Future<U> x : futureList) {
U taskRet = x.get();
if (taskRet != null) {
failRecord.add(taskRet);
}
}
double percent = ((finalI + 1.0) * pageSize > count ? count : (finalI + 1.0) * pageSize) / count;
log.info("PageTaskUtil.exe 已处理{}页,共{}页,进度{}%,预计剩余时间{}s", finalI + 1, Math.ceil(count * 1.0 / pageSize), String.format("%.2f", percent * 100), String.format("%.2f", timer.intervalSecond() / percent * (1 - percent)));
} catch (Exception e) {
log.error("PageTaskUtil.exe error:", e);
}
return failRecord;
};
//处理数据
FutureTask<ArrayList<U>> task = new FutureTask<>(producer);
futureListProducer.add(task);
executorService.execute(task);
}
//获取处理数据的返回值
for (Future<ArrayList<U>> x : futureListProducer) {
ArrayList<U> taskRet = null;
try {
taskRet = x.get();
} catch (InterruptedException | ExecutionException e) {
throw new RuntimeException(e);
}
failRecords.addAll(taskRet);
}
String ret = "Done,but failed Ids:\n" + CollUtil.join(failRecords, ",");
log.info("PageTaskUtil.exe 已完成!共处理{}条,用时{}s.\n{}", count, timer.intervalSecond(), ret);
return ret;
}
/**
* 分页异步执行,生产者为分页主线程执行,消费者为异步线程池执行
* @param count 生产者记录总数
* @param pageSize 分页大小
* @param getList 获取生产者数据,入参为 分页开始索引
* @param process 消费数据
* @param errorFun 出现异常时用返回数据
* @return 返回出错的数据按,分割
*/
public static <T, U> String exe(Long count, Integer pageSize, Function<Integer,List<T>> getList, Consumer<T> process, Function<T,U> errorFun){
Function<T,Callable<U>> callableFunction=x->()->{
try {
process.accept(x);
return null;
} catch (Exception e) {
log.error("PageTaskUtil.exe runnable error:",e);
log.error("PageTaskUtil.exe current Record:\n{}", JSONUtil.toJsonStr(x));
return errorFun.apply(x);
}
};
return exe(count, pageSize, getList, callableFunction);
}
/**
* 分页异步执行,生产者为分页主线程执行,消费者为异步线程池执行
* @param count 生产者记录总数
* @param pageSize 分页大小
* @param getList 获取生产者数据,入参为 分页开始索引
* @param process 消费数据
*/
public static <T, U> void exe(Long count, Integer pageSize, Function<Integer,List<T>> getList, Consumer<T> process){
Function<T,Callable<U>> callableFunction=x->()->{
try {
process.accept(x);
} catch (Exception e) {
log.error("PageTaskUtil.exe runnable error:",e);
log.error("PageTaskUtil.exe current Record:\n{}", JSONUtil.toJsonStr(x));
}
return null;
};
exe(count, pageSize, getList, callableFunction);
}
}