多线程分批处理数据

在工作中经常会遇到一个业务操作要处理成千上万的数据，耗时非常严重，性能严重不达标，这个时候就会想到采用多线程来执行业务来提高执行效率，在改背景下本人写了一个可以直接套用业务的demo，在此记录，以备随时用到来copy代码

首先定义个执行业务任务的接口

package AtomicTest;

/**
 * desc:
 * author:wangqiangac
 * date:2023/12/21 14:48
 */
public interface MyTask {

    <T> String  work(T www) throws Exception;
}

再创建一个构建线程池的工具类，代码如下，线程池参数可以根据自己需要做成可配置

package AtomicTest;

import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;

import java.util.concurrent.RejectedExecutionHandler;
import java.util.concurrent.ThreadPoolExecutor;

/**
 * desc:
 * author:wangqiangac
 * date:2023/12/21 17:17
 */
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class ThreadPoolBuilder {
    int corePoolSize;//设置ThreadPoolExecutor的核心池大小。默认值为1。此设置可以在运行时进行修改，例如通过JMX进行修改
    int maxPoolSize;//设置ThreadPoolExecutor的最大池大小。默认值为Integer。最大值。此设置可以在运行时进行修改，例如通过JMX进行修改。
    int queueCapacity;//设置ThreadPoolExecutor的BlockingQueue的容量。默认值为Integer。最大值
    int keepAliveSeconds;//设置ThreadPoolExecutor的保持活动秒数。默认值为60。
    String threadNamePrefix;//自定义名称前缀
    boolean prestartAllCoreThreads;//指定是否启动所有核心线程，使它们空闲等待工作。
    RejectedExecutionHandler rejectedExecutionHandler;
    public ThreadPoolBuilder setCorePoolSize(int corePoolSize) {
        this.corePoolSize = corePoolSize;
        return this;
    }

    public ThreadPoolBuilder setMaxPoolSize(int maxPoolSize) {
        this.maxPoolSize = maxPoolSize;
        return this;
    }

    public ThreadPoolBuilder setQueueCapacity(int queueCapacity) {
        this.queueCapacity = queueCapacity;
        return this;
    }

    public ThreadPoolBuilder setKeepAliveSeconds(int keepAliveSeconds) {
        this.keepAliveSeconds = keepAliveSeconds;
        return this;
    }

    public ThreadPoolBuilder setThreadNamePrefix(String threadNamePrefix) {
        this.threadNamePrefix = threadNamePrefix;
        return this;
    }

    public ThreadPoolBuilder setPrestartAllCoreThreads(boolean prestartAllCoreThreads) {
        this.prestartAllCoreThreads = prestartAllCoreThreads;
        return this;
    }

    public ThreadPoolBuilder setRejectedExecutionHandler(RejectedExecutionHandler rejectedExecutionHandler) {
        this.rejectedExecutionHandler = rejectedExecutionHandler;
        return this;
    }

    public ThreadPoolExecutor builderThreadPoolExecutor(){
        ThreadPoolTaskExecutor threadPoolTaskExecutor = new ThreadPoolTaskExecutor();
        threadPoolTaskExecutor.setCorePoolSize(corePoolSize);
        threadPoolTaskExecutor.setMaxPoolSize(maxPoolSize);
        threadPoolTaskExecutor.setQueueCapacity(queueCapacity);
        threadPoolTaskExecutor.setKeepAliveSeconds(keepAliveSeconds);
        threadPoolTaskExecutor.setThreadNamePrefix(threadNamePrefix);
        threadPoolTaskExecutor.setPrestartAllCoreThreads(prestartAllCoreThreads);
        threadPoolTaskExecutor.setRejectedExecutionHandler(rejectedExecutionHandler);
        threadPoolTaskExecutor.initialize();
        ThreadPoolExecutor threadPoolExecutor = threadPoolTaskExecutor.getThreadPoolExecutor();
        return threadPoolExecutor;
    }
}

然后创建一个用于执行批量计算的工具类，如下

package AtomicTest;

import lombok.extern.slf4j.Slf4j;

import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.stream.Collectors;
import java.util.stream.IntStream;

/**
 * desc:
 * author:wangqiangac
 * date:2023/12/22 15:56
 */
@Slf4j
public class ThreadUtil {
    /**
     * 按照批次处理任务
     * @param threadPoolExecutor
     * @param list
     * @param batchCount
     */
    public static <T> List<List<T>> bathCalc(ThreadPoolExecutor threadPoolExecutor, List<T> list,int batchCount,String taskName,MyTask myTask) throws InterruptedException, ExecutionException {
        List<Callable<Object>> callables = new ArrayList<>();
        List<List<T>> partList = IntStream.range(0, list.size())
                .boxed()
                .collect(Collectors.groupingBy(index -> index / batchCount))
                .values()
                .stream()
                .map(indices -> indices.stream().map(list::get).collect(Collectors.toList()))
                .collect(Collectors.toList());
        int batch = 0;
        log.error("{}一共拆分为{}批次开始执行",taskName,partList.size());
        for (List<T> splitList :partList) {
            batch++;
            int finalBatch = batch;
            Instant readyStart = Instant.now();
            callables.add(new Callable<Object>() {
                @Override
                public Object call() throws Exception {
                    log.error("{}第{}批开始执行", taskName,finalBatch);
                    Instant start = Instant.now();
                    List<T> results = new ArrayList<>();
                    for (T t : splitList) {
                        T result = (T) myTask.work(t);
                        results.add(result);
                    }
                    Instant end = Instant.now();
                    log.error("{}第{}批执行结束,线程等待时间为{}(ms),当前批次执行耗时{}(ms)", taskName,finalBatch,Duration.between(readyStart,start).toMillis(),Duration.between(start,end).toMillis());
                    return results;
                }
            });
        }
        List<Future<Object>> futures = threadPoolExecutor.invokeAll(callables);
        List<List<T>> results = new ArrayList<>();
        for (Future future : futures) {
            List<T> o = (List<T>) future.get();
            results.add(o);
        }
        return results;
    }
}

下面代码为调用代码，调用逻辑为定义了一批data英雄分批次开启多线程进入战场大乱斗，最后各个英雄的耗时及战绩

package AtomicTest;

import disruptor.WwwApplication;
import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.api.Test;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import org.springframework.util.CollectionUtils;

import java.io.InputStream;
import java.time.Duration;
import java.time.Instant;
import java.util.*;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import java.util.stream.Stream;


/**
 * desc:
 * author:wangqiangac
 * date:2023/12/21 14:22
 */
@Slf4j
@SpringBootTest(classes=WwwApplication.class)
public class ThreadPoolTest {
    public static  List<String> list = new ArrayList<>();
    static{
        list.add("斧王");list.add("剑圣");
        list.add("小黑");list.add("幽鬼");
        list.add("火枪");list.add("大屁股");
        list.add("哈市卡");list.add("虚空假面");
        list.add("蓝胖");list.add("路西法");
        list.add("美杜莎");list.add("巫医");
        list.add("莱恩");list.add("冰女");
        list.add("幻影刺客");list.add("圣堂刺客");
        list.add("火女");list.add("潮汐");
        list.add("赏金猎人");list.add("蓝猫");
    }
    @Test
    public void test() throws InterruptedException, ExecutionException {
        ThreadPoolBuilder threadPoolBuilder = new ThreadPoolBuilder();
        threadPoolBuilder.setCorePoolSize(5).setMaxPoolSize(20).setKeepAliveSeconds(120).setQueueCapacity(20)
                .setThreadNamePrefix("王强的线程").setPrestartAllCoreThreads(true).setRejectedExecutionHandler(new BlockPolicy());
        ThreadPoolExecutor threadPoolExecutor = threadPoolBuilder.builderThreadPoolExecutor();

        List<List<String>> results = ThreadUtil.bathCalc(threadPoolExecutor, list, 5, "决战冰封之巅任务",new MyTask() {
            @Override
            public <T> String work(T www) throws Exception {
                Instant start = Instant.now();
                log.error(www.toString()+"切入战场");
                Thread.sleep(new Random().nextInt(20) * 1000);
                Instant end = Instant.now();
                long l = Duration.between(start, end).toMillis();
                log.error(www.toString()+"战场耗时(ms):"+l);
                return www.toString() + "获得" + new Random().nextInt(10) + "血";
            }
        });
        log.error("各个英雄最终获得成就如下:");
        for (List<String> list1:results) {
            log.error("当前队伍获取成绩如下:");
            for (String name: list1) {
                log.error(name);
            }
        }

    }
}

最终运行结果如下