通过配置类定义线程池

zeki豪

已于 2022-10-24 16:28:33 修改

阅读量311

点赞数

分类专栏：实操文章标签： 1024程序员节 java

于 2022-10-24 16:24:06 首次发布

本文链接：https://blog.csdn.net/EDT777/article/details/127494632

版权

实操专栏收录该内容

9 篇文章 0 订阅

订阅专栏

文章目录

线程工具类
yml配置文件定义线程池参数
线程池配置类(供参考)

线程工具类

ThreadUtil

public class ThreadUtil {

    private final static Logger logger = LoggerFactory.getLogger(ThreadUtil.class);

    public static void sleepByWait(long timeout) throws InterruptedException {
        byte[] lock = new byte[0];
        synchronized (lock){
            lock.wait(timeout);
        }
        lock = null;
    }


    public static RejectedExecutionHandler blockExecuteRejectHandle(String name){
        return new BlockExecuteRejectHandle(name);
    }

    public static ThreadFactory threadFactory(String name){
        return new TFactory(name);
    }

    static class TFactory implements ThreadFactory {

        private final ThreadGroup group;
        //编号
        private final AtomicInteger threadNumber = new AtomicInteger(1);
        //池子名称
        private final String namePrefix;

        public TFactory(String name) {
            SecurityManager s = System.getSecurityManager();
            group = (s != null) ? s.getThreadGroup() :
                    Thread.currentThread().getThreadGroup();
            namePrefix = name.concat("-");
        }

        public Thread newThread(Runnable r) {
            Thread t = new Thread(group, r,
                    namePrefix + threadNumber.getAndIncrement(),
                    0);
            if (t.isDaemon())
                t.setDaemon(false);
            if (t.getPriority() != Thread.NORM_PRIORITY)
                t.setPriority(Thread.NORM_PRIORITY);
            return t;
        }
    }


    static class BlockExecuteRejectHandle implements RejectedExecutionHandler {

        final String name;

        public BlockExecuteRejectHandle(String name) {
            this.name = name.concat("RejectHandle");
        }

        @Override
        public void rejectedExecution(Runnable r, ThreadPoolExecutor e) {
            if (!e.isShutdown()) {
                try {
                    logger.warn("{} 阻塞加入 | pool:{}",name,e);
                    e.getQueue().put(r);
                } catch (Exception ex) {
                    logger.error("{} 阻塞加入异常",e,ex);
                }
            }
        }
    }

    /**
     * 每个线程任务需要处理的数据量
     * @param dataSize 数据个数
     * @param maxTaskNum 最多创建多少个线程任务
     * @return  返回<=1  就再当前线程执行    返回其他就在池子中执行
     */
    public static int avgCapacity(int dataSize, int maxTaskNum) {
        int _c = dataSize / maxTaskNum;
        if (_c == 0){
            return 0;//dataSize < maxTaskNum 就不使用线程池执行
        }
        return dataSize % maxTaskNum == 0 ? _c : _c + 1;
    }

    /**
     * 某个大集合切割为多个
     * @param ls 数据集
     * @param fun 需要并发执行的方法
     * @param executor 线程池
     * @param maxTaskNum 此集合最多拆为几个任务
     * @param <T>
     */
    public static <T> void concurrentExecuteAndBlockResult(ThreadPoolExecutor executor, int maxTaskNum,Function<List<T>, Void> fun,List<T> ls){
        if (ls.isEmpty()){
            return;
        }
        int avgCapacity = ThreadUtil.avgCapacity(ls.size(), maxTaskNum);
        if (avgCapacity <= 1){
            fun.apply(ls);
        }else {
            List<List<T>> lists = Lists.partition(ls, avgCapacity);
            CompletableFuture[] all = new CompletableFuture[lists.size()];
            for (int i = 0; i < lists.size(); i++) {
                List<T> tmp = lists.get(i);
                if (tmp.isEmpty()){
                    continue;
                }
                all[i] = CompletableFuture.runAsync(() -> fun.apply(tmp), executor);
            }
            CompletableFuture.allOf(all).join();
        }
    }

    /**
     * 某个大集合切割为多个
     * @param ls 数据集
     * @param fun 需要并发执行的方法
     * @param executor 线程池
     * @param oneTaskDataSize 一个任务多少条数据
     * @param <T>
     */
    public static <T> void concurrentExecuteAndBlockResultVo(ThreadPoolExecutor executor, int oneTaskDataSize,Function<List<T>, Void> fun,List<T> ls){
        if (ls.isEmpty()){
            return;
        }
        if (ls.size() <= oneTaskDataSize){
            fun.apply(ls);
        }else {
            List<List<T>> lists = Lists.partition(ls, oneTaskDataSize);
            CompletableFuture[] all = new CompletableFuture[lists.size()];
            for (int i = 0; i < lists.size(); i++) {
                List<T> tmp = lists.get(i);
                if (tmp.isEmpty()){
                    continue;
                }
                all[i] = CompletableFuture.runAsync(() -> fun.apply(tmp), executor);
            }
            CompletableFuture.allOf(all).join();
        }
    }


    /**
     * 某个大集合切割为多个
     * @param ls 数据集
     * @param fun 需要并发执行的方法
     * @param executor 线程池
     * @param oneTaskDataSize 一个任务多少条数据
     * @param <T>
     */
    public static <T> void concurrentExecuteAndBlockResultVoForAbortPolicyReject(ThreadPoolExecutor executor, int oneTaskDataSize, Function<List<T>, Void> fun, List<T> ls){
        if (ls.isEmpty()){
            return;
        }
        if (ls.size() <= oneTaskDataSize){
            fun.apply(ls);
        }else {
            List<List<T>> lists = Lists.partition(ls, oneTaskDataSize);
            CompletableFuture[] all = new CompletableFuture[lists.size()];
            for (int i = 0; i < lists.size(); i++) {
                List<T> tmp = lists.get(i);
                if (tmp.isEmpty()){
                    continue;
                }
                int reNum = 0;
                while (all[i] == null){
                    try {
                        all[i] = CompletableFuture.runAsync(() -> fun.apply(tmp), executor);
                    } catch (RejectedExecutionException e) {
                        if (reNum == 0){
                            logger.warn("线程池处理任务繁忙:{}",e.getMessage());
                        }
                        reNum++;
                        try {Thread.sleep(3);} catch (Exception e1) {}
                    }catch (Exception e){
                        logger.error("线程池处理任务异常",e);
                        break;
                    }
                }

                if (reNum>0){
                    logger.warn("线程池处理任务繁忙 重试次数:{}",reNum);
                }
            }
            CompletableFuture.allOf(all).join();
        }
    }

    /**
     * 某个大集合切割为多个
     * @param ls 数据集
     * @param fun 需要并发执行的方法
     * @param executor 线程池
     * @param maxTaskNum 此集合最多拆为几个任务
     * @param <T>
     */
    public static <T> void concurrentExecuteAndBlockResult(ThreadPoolExecutor executor, int maxTaskNum,BiFunction<List<T>, Object, Void> fun, List<T> ls,Object p0){
        if (ls.isEmpty()){
            return;
        }
        int avgCapacity = ThreadUtil.avgCapacity(ls.size(), maxTaskNum);
        if (avgCapacity <= 1){
            fun.apply(ls,p0);
        }else {
            List<List<T>> lists = Lists.partition(ls, avgCapacity);
            CompletableFuture[] all = new CompletableFuture[lists.size()];
            for (int i = 0; i < lists.size(); i++) {
                List<T> tmp = lists.get(i);
                if (tmp.isEmpty()){
                    continue;
                }
                all[i] = CompletableFuture.runAsync(() -> fun.apply(tmp,p0), executor);
            }
            CompletableFuture.allOf(all).join();
        }
    }

    /**
     * 某个大集合切割为多个
     * @param ls 数据集
     * @param fun 需要并发执行的方法
     * @param executor 线程池
     * @param maxTaskNum 此集合最多拆为几个任务
     * @param <T>
     */
    public static <T> void concurrentExecuteAndBlockResult(ThreadPoolExecutor executor, int maxTaskNum,BiFunction<List<T>, Object[], Void> fun, List<T> ls,Object... oArr){
        if (ls.isEmpty()){
            return;
        }
        int avgCapacity = ThreadUtil.avgCapacity(ls.size(), maxTaskNum);
        if (avgCapacity <= 1){
            fun.apply(ls,oArr);
        }else {
            List<List<T>> lists = Lists.partition(ls, avgCapacity);
            CompletableFuture[] all = new CompletableFuture[lists.size()];
            for (int i = 0; i < lists.size(); i++) {
                List<T> tmp = lists.get(i);
                if (tmp.isEmpty()){
                    continue;
                }
                all[i] = CompletableFuture.runAsync(() -> fun.apply(tmp,oArr), executor);
            }
            CompletableFuture.allOf(all).join();
        }
    }



    public static <T,R> List<R> exec(ThreadPoolExecutor executor, int maxTaskNum,
                                     Function<List<T>, List<R>> fun,
                                     List<T> dataLs){
        if (dataLs.isEmpty()){
            return new ArrayList<>();
        }
        int avgCapacity = avgCapacity(dataLs.size(), maxTaskNum);
        if (avgCapacity <= 1){
            return fun.apply(dataLs);
        }else {
            List<R> ret = new CopyOnWriteArrayList<>();
            List<List<T>> lists = Lists.partition(dataLs, avgCapacity);
            CompletableFuture<? extends List<? extends R>>[] all = new CompletableFuture[lists.size()];
            for (int i = 0; i < lists.size(); i++) {
                List<T> tmp = lists.get(i);
                if (tmp.isEmpty()){
                    continue;
                }
                all[i] = CompletableFuture.supplyAsync(() -> fun.apply(tmp), executor).whenCompleteAsync((rv, ex) -> {
                    if (ex != null) {
                        ex.printStackTrace();
                    }
                    if (rv != null) {
                        ret.addAll(rv);
                    }
                });
            }
            CompletableFuture.allOf(all).join();
            return ret;
        }
    }

}

yml配置文件定义线程池参数

# 业务线程池配置
business-thread-pools:
  # 国内短信
  sms:
    # 从各个网关采集状态报告线程池，建议值为网关的个数
    collect-report-pool:
      pool-core-size: 2

线程池配置类(供参考)


import com.xyc.sms.common.util.ThreadUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

@Configuration
public class BusinessThreadPoolConfig {

    private final static Logger logger = LoggerFactory.getLogger(BusinessThreadPoolConfig.class);

    /**
     * 引用外部yml配置文件参数(外部没配置参数就用`PoolConfig`自定义默认的参数)
     */
    @Bean(name = "collectReportPoolConfig")
    @ConfigurationProperties(prefix = "business-thread-pools.sms.collect-report-pool")
    public PoolConfig collectReportPoolConfig(){
        return new PoolConfig("CollectReportPool");
    }


    /**
     * 不引用外部Yml配置参数的写法(线程池参数在`PoolConfig`类里面自己写死)
     */
    @Bean(name = "queueConfig")
    public PoolConfig queueTransferSqlConfig() {
        return new PoolConfig("QueueTransferSql");
    }


    /**
     * 拒绝策略使用等待队列的线程池配置
     */
    @Bean(name = "timToQueueThreadPool")
    public ThreadPoolExecutor timToQueueThreadPool(@Qualifier("queueConfig") PoolConfig c) {
        //打印参数日志
        PoolConfig useConfig = c.printInitInfoAndReturnUse(true);
        return new ThreadPoolExecutor(useConfig.getPoolCoreSize(), useConfig.getPoolMaxSize(), 60, TimeUnit.SECONDS,
                new LinkedBlockingQueue<>(),
                ThreadUtil.threadFactory(useConfig.getPoolNamPrefix()),
                new ThreadPoolExecutor.CallerRunsPolicy());
    }


    /**
     * 采集状态报告 线程池
     */
    @Bean(name = "collectReportPool")
    public ThreadPoolExecutor collectReportPool(@Qualifier("collectReportPoolConfig") PoolConfig c){
        PoolConfig useConfig = c.printInitInfoAndReturnUse(true);//打印参数日志
        return new ThreadPoolExecutor(useConfig.getPoolCoreSize(),useConfig.getPoolMaxSize(),0, TimeUnit.MILLISECONDS,
                new LinkedBlockingQueue<>(),
                ThreadUtil.threadFactory(useConfig.getPoolNamPrefix()),
                ThreadUtil.blockExecuteRejectHandle(useConfig.getPoolNamPrefix()));
    }



    /**
     * 线程池配置类
     */
    private class PoolConfig {

        private int poolCoreSize = -1;
        private int poolMaxSize = -1;
        private int taskQueueMaxSize = -1;
        private String poolNamPrefix;

        //不配置使用的默认值  ps：如果不配置 yml
        private int poolCoreSize_default = 1;
        private int poolMaxSize_default = 2;
        private int taskQueueMaxSize_default = 1;

        public PoolConfig(String poolNamPrefix) {
            this.poolNamPrefix = poolNamPrefix;
        }

        public int getPoolCoreSize() {
            return poolCoreSize;
        }

        public void setPoolCoreSize(int poolCoreSize) {
            this.poolCoreSize = poolCoreSize;
        }

        public int getPoolMaxSize() {
            return poolMaxSize;
        }

        public void setPoolMaxSize(int poolMaxSize) {
            this.poolMaxSize = poolMaxSize;
        }

        public int getTaskQueueMaxSize() {
            return taskQueueMaxSize;
        }

        public void setTaskQueueMaxSize(int taskQueueMaxSize) {
            this.taskQueueMaxSize = taskQueueMaxSize;
        }

        public String getPoolNamPrefix() {
            return poolNamPrefix;
        }

        public void setPoolNamPrefix(String poolNamPrefix) {
            this.poolNamPrefix = poolNamPrefix;
        }

        public int getPoolCoreSize_default() {
            return poolCoreSize_default;
        }

        public void setPoolCoreSize_default(int poolCoreSize_default) {
            this.poolCoreSize_default = poolCoreSize_default;
        }

        public int getPoolMaxSize_default() {
            return poolMaxSize_default;
        }

        public void setPoolMaxSize_default(int poolMaxSize_default) {
            this.poolMaxSize_default = poolMaxSize_default;
        }

        public int getTaskQueueMaxSize_default() {
            return taskQueueMaxSize_default;
        }

        public void setTaskQueueMaxSize_default(int taskQueueMaxSize_default) {
            this.taskQueueMaxSize_default = taskQueueMaxSize_default;
        }

        private PoolConfig printInitInfoAndReturnUse(boolean isOnlyUseCore){
            PoolConfig us = new PoolConfig(this.poolNamPrefix);
            logger.info(String.format("****************************************%-5s初始化【%-25s】线程池配置项 开始%-5s****************************************","",us.getPoolNamPrefix(),""));
            if (isOnlyUseCore){
                //只有核心配置  那么max也使用核心配置
                us.setPoolCoreSize(this.poolCoreSize == -1?this.poolCoreSize_default:this.poolCoreSize);
                us.setPoolMaxSize(us.getPoolCoreSize());
                us.setTaskQueueMaxSize(Integer.MAX_VALUE);
                logger.info(String.format("**************%-4s 加载 %-25s - %-20s  默认值: %-8d yml配置: %-8d 启动使用配置: %-5d**************",
                        "",us.getPoolNamPrefix(),"poolCoreSize", us.getPoolCoreSize_default(),this.poolCoreSize,us.getPoolCoreSize()));
                logger.info(String.format("**************%-4s 加载 %-25s - %-20s  默认值: %-8d yml配置: %-8d 启动使用配置(使用core配置项): %-5d**************",
                        "",us.getPoolNamPrefix(),"poolMaxSize", us.getPoolMaxSize_default(), this.poolMaxSize,us.getPoolMaxSize()));
            }else {
                us.setPoolCoreSize(this.poolCoreSize == -1?this.poolCoreSize_default:this.poolCoreSize);
                us.setPoolMaxSize(this.poolMaxSize == -1?this.poolMaxSize_default:this.poolMaxSize);
                us.setTaskQueueMaxSize(this.taskQueueMaxSize == -1 ?this.taskQueueMaxSize_default : this.taskQueueMaxSize);
                logger.info(String.format("**************%-4s 加载 %-25s - %-20s  默认值: %-8d yml配置: %-8d 启动使用配置: %-5d**************",
                        "",us.getPoolNamPrefix(),"poolCoreSize", us.getPoolCoreSize_default(),this.poolCoreSize,us.getPoolCoreSize()));
                logger.info(String.format("**************%-4s 加载 %-25s - %-20s  默认值: %-8d yml配置: %-8d 启动使用配置: %-5d**************",
                        "",us.getPoolNamPrefix(),"poolMaxSize",us.getPoolMaxSize_default(), this.poolMaxSize,us.getPoolMaxSize()));
                logger.info(String.format("**************%-4s 加载 %-25s - %-20s  默认值: %-8d yml配置: %-8d 启动使用配置: %-5d**************",
                        "",us.getPoolNamPrefix(),"taskQueueMaxSize", us.getTaskQueueMaxSize_default(), this.taskQueueMaxSize,us.getTaskQueueMaxSize()));
            }
            logger.info(String.format("****************************************%-5s初始化【%-25s】线程池配置项 结束%-5s****************************************","",us.getPoolNamPrefix(),""));
            return us;
        }

    }

}