【动态线程池组件开发】

赚够壹仟万

已于 2024-06-09 13:10:47 修改

阅读量623

点赞数 24

分类专栏： Spring中间件开发文章标签： java spring maven spring boot 中间件

于 2024-06-09 13:10:01 首次发布

本文链接：https://blog.csdn.net/weixin_44794897/article/details/139560645

版权

Spring中间件开发专栏收录该内容

4 篇文章 0 订阅

订阅专栏

概要

线程池使⽤⾯临的核⼼的问题在于：线程池的参数并不好配置。配置合理需要强依赖开发⼈员的个⼈经验和知识；另⼀⽅⾯，线程池执⾏的情况和任务类型相关性较⼤，IO密集型和CPU密集型的任务运⾏起来的情况差异⾮常⼤，这导致业界并没有⼀些成熟的经验策略帮助开发⼈员参考。既然不能保证参数配置最合理，那么是否可以通过将修改线程池参数的成本降下来，这样⾄少可以发⽣故障的时候可以快速调整从⽽缩短故障恢复的时间？基于这个思考，我们可以将线程池的参数从代码中迁移到注册中心上，实现线程池参数可动态配置和即时⽣效。

整体架构流程

整体框架流程开发动态线程池SpringBoot Starter组件，通过该组件实现线程池参数信息可动态配置和即时生效

中间件层结构

在中间件实现工程分为四块，config、domain、registry，trigger它们的作用如下：

config，读取自定义配置信息，以及负责把注册中心redis相关的Bean、spring相关的bean 加载启动 (包括动态注册线程池信息到注册中心的自定义bean，以及操作线程池的自定义bean)。
domain，这部分是动态线程池服务，定义了线程池配置实体对象、注册中心枚举值对象，对外提供IDynamicThreadPoolService进行线程池数据查询、线程池参数查询、更新线程池参数的服务。
registry，这一部分是注册线程池配置信息、线程池参数到注册中心，线程池配置信息使用list，线程池参数使用缓存，存成Bucket.
trigger,这部分主要实现线程池数据的定时上报job 、以及线程池配置参数变更的redis消息触发。

实现步骤

创建DynamicThreadPoolAutoConfig类，这是Spring Boot的自动配置类，用于初始化和配置动态线程池。在DynamicThreadPoolAutoConfig类中，定义了多个Bean，包括redissonClient、redisRegistry、dynamicThreadPoolService、threadPoolDataReportJob、threadPoolConfigAdjustListener和dynamicThreadPoolRedisTopic。这些Bean分别用于创建Redis客户端，注册中心，动态线程池服务，线程池数据报告任务，线程池配置调整监听器和Redis主题。

@Configuration
@EnableConfigurationProperties(DynamicThreadPoolAutoProperties.class)
@EnableScheduling
public class DynamicThreadPoolAutoConfig {

    private final Logger logger = LoggerFactory.getLogger(DynamicThreadPoolAutoConfig.class);

    private String applicationName;

    @Bean("redissonClient")
    public RedissonClient redissonClient(DynamicThreadPoolAutoProperties properties){
        Config config = new Config();
        config.setCodec(JsonJacksonCodec.INSTANCE);
        config.useSingleServer()
                .setAddress("redis://" + properties.getHost() + ":" + properties.getPort())
                .setPassword(properties.getPassword())
                .setConnectionPoolSize(properties.getPoolSize())
                .setConnectionMinimumIdleSize(properties.getMinIdleSize())
                .setIdleConnectionTimeout(properties.getConnectTimeout())
                .setConnectTimeout(properties.getConnectTimeout())
                .setRetryAttempts(properties.getRetryAttempts())
                .setRetryInterval(properties.getRetryInterval())
                .setPingConnectionInterval(properties.getPingInterval())
                .setKeepAlive(properties.isKeepAlive())
        ;
        RedissonClient redissonClient = Redisson.create(config);
        return redissonClient;
    }

    // 初始化注册中心
    @Bean
    public IRegistry redisRegistry(RedissonClient redissonClient){
        return new RedisRegistry(redissonClient);
    }

    @Bean("dynamicThreadPoolService")
    public DynamicThreadPoolService dynamicThreadPoolService(ApplicationContext applicationContext, Map<String,ThreadPoolExecutor> threadPoolExecutorMap,RedissonClient redissonClient){
        applicationName = applicationContext.getEnvironment().getProperty("spring.application.name");

        if (StringUtils.isBlank(applicationName)) {
            applicationName = "缺省的";
            logger.warn("动态线程池，启动提示。SpringBoot 应用未配置 spring.application.name 无法获取到应用名称！");
        }

        // 防止应用启动时 已经配置了线程池的信息
        Set<String> keySet = threadPoolExecutorMap.keySet();
        for (String threadPoolKey : keySet) {
            ThreadPoolConfigEntity threadPoolConfig = redissonClient.<ThreadPoolConfigEntity>getBucket(RegistryEnumVO.THREAD_POOL_CONFIG_PARAMETER_LIST_KEY.getKey() + "_" + applicationName + "_" + threadPoolKey).get();
            if(null == threadPoolConfig) continue;
            ThreadPoolExecutor executor = threadPoolExecutorMap.get(threadPoolKey);
            executor.setCorePoolSize(threadPoolConfig.getCorePoolSize());
            executor.setMaximumPoolSize(threadPoolConfig.getMaximumPoolSize());
        }

        return new DynamicThreadPoolService(applicationName,threadPoolExecutorMap);
    }

    // 创建动态注册组件任务.... 定时将数据库信息刷新到redis中...
    @Bean
    public ThreadPoolDataReportJob threadPoolDataReportJob(IDynamicThreadPoolService dynamicThreadPoolService,IRegistry registry){
        return new ThreadPoolDataReportJob(dynamicThreadPoolService,registry);
    }

    @Bean
    public ThreadPoolConfigAdjustListener threadPoolConfigAdjustListener(IDynamicThreadPoolService dynamicThreadPoolService, IRegistry registry) {
        return new ThreadPoolConfigAdjustListener(dynamicThreadPoolService, registry);
    }


    // 添加一个监听
    @Bean(name = "dynamicThreadPoolRedisTopic")
    public RTopic threadPoolConfigAdjustListener(RedissonClient redissonClient,ThreadPoolConfigAdjustListener threadPoolConfigAdjustListener){
        RTopic topic = redissonClient.getTopic(RegistryEnumVO.DYNAMIC_THREAD_POOL_REDIS_TOPIC.getKey() + "_" + applicationName);
        topic.addListener(ThreadPoolConfigEntity.class,threadPoolConfigAdjustListener);
        return topic;
    }
}

其中DynamicThreadPoolAutoProperties用来读取使用了该组件的应用配置的信息

# 动态线程池管理配置
dynamic:
  thread:
    pool:
      config:
        # 状态；true = 开启、false 关闭
        enabled: true
        # redis host
        host: 127.0.0.1
        # redis port
        port: 6379

DynamicThreadPoolAutoProperties类

@ConfigurationProperties(prefix = "dynamic.thread.pool.config",ignoreInvalidFields = true)
public class DynamicThreadPoolAutoProperties {

    /** 状态；open = 开启、close 关闭 */
    private boolean enable;
    /** redis host */
    private String host;
    /** redis port */
    private int port;
    /** 账密 */
    private String password;
    /** 设置连接池的大小，默认为64 */
    private int poolSize = 64;
    /** 设置连接池的最小空闲连接数，默认为10 */
    private int minIdleSize = 10;
    /** 设置连接的最大空闲时间（单位：毫秒），超过该时间的空闲连接将被关闭，默认为10000 */
    private int idleTimeout = 10000;
    /** 设置连接超时时间（单位：毫秒），默认为10000 */
    private int connectTimeout = 10000;
    /** 设置连接重试次数，默认为3 */
    private int retryAttempts = 3;
    /** 设置连接重试的间隔时间（单位：毫秒），默认为1000 */
    private int retryInterval = 1000;
    /** 设置定期检查连接是否可用的时间间隔（单位：毫秒），默认为0，表示不进行定期检查 */
    private int pingInterval = 0;
    /** 设置是否保持长连接，默认为true */
    private boolean keepAlive = true;

    public boolean isEnable() {
        return enable;
    }

    public void setEnable(boolean enable) {
        this.enable = enable;
    }

    public String getHost() {
        return host;
    }

    public void setHost(String host) {
        this.host = host;
    }

    public int getPort() {
        return port;
    }

    public void setPort(int port) {
        this.port = port;
    }

    public String getPassword() {
        return password;
    }

    public void setPassword(String password) {
        this.password = password;
    }

    public int getPoolSize() {
        return poolSize;
    }

    public void setPoolSize(int poolSize) {
        this.poolSize = poolSize;
    }

    public int getMinIdleSize() {
        return minIdleSize;
    }

    public void setMinIdleSize(int minIdleSize) {
        this.minIdleSize = minIdleSize;
    }

    public int getIdleTimeout() {
        return idleTimeout;
    }

    public void setIdleTimeout(int idleTimeout) {
        this.idleTimeout = idleTimeout;
    }

    public int getConnectTimeout() {
        return connectTimeout;
    }

    public void setConnectTimeout(int connectTimeout) {
        this.connectTimeout = connectTimeout;
    }

    public int getRetryAttempts() {
        return retryAttempts;
    }

    public void setRetryAttempts(int retryAttempts) {
        this.retryAttempts = retryAttempts;
    }

    public int getRetryInterval() {
        return retryInterval;
    }

    public void setRetryInterval(int retryInterval) {
        this.retryInterval = retryInterval;
    }

    public int getPingInterval() {
        return pingInterval;
    }

    public void setPingInterval(int pingInterval) {
        this.pingInterval = pingInterval;
    }

    public boolean isKeepAlive() {
        return keepAlive;
    }

    public void setKeepAlive(boolean keepAlive) {
        this.keepAlive = keepAlive;
    }
}

a、redissonClient Bean用于创建Redis客户端，用于与Redis服务器进行通信。
b、redisRegistry Bean用于创建注册中心，用于报告线程池的状态。
c、dynamicThreadPoolService Bean用于创建动态线程池服务，用于管理线程池。
d、threadPoolDataReportJob Bean用于创建线程池数据报告任务，用于定期向注册中心报告线程池的状态。
e、threadPoolConfigAdjustListener Bean用于创建线程池配置调整监听器，用于监听Redis主题上的消息，当收到消息时，调整线程池的配置。
f、dynamicThreadPoolRedisTopic Bean用于创建Redis主题，用于发布和订阅消息。

创建IRegistry接口，定义了报告线程池的方法。创建RedisRegistry类，实现了IRegistry接口，使用Redis作为注册中心，实现了报告线程池的方法。

public interface IRegistry {

    // 上报线程池
    void reportThreadPool(List<ThreadPoolConfigEntity> threadPoolConfigEntites);

    // 上报线程池配置参数
    void reportThreadPoolConfigParameter(ThreadPoolConfigEntity threadPoolConfigEntity);

}

接口的实现类

public class RedisRegistry implements IRegistry {

    private final RedissonClient redissonClient;

    public RedisRegistry(RedissonClient redissonClient) {
        this.redissonClient = redissonClient;
    }

    @Override
    public void reportThreadPool(List<ThreadPoolConfigEntity> threadPoolConfigEntities) {
        RList<ThreadPoolConfigEntity> list = redissonClient.getList(RegistryEnumVO.THREAD_POOL_CONFIG_LIST_KEY.getKey());
        list.delete();
        list.addAll(threadPoolConfigEntities);
    }

    // 上报单个线程池变化信息
    @Override
    public void reportThreadPoolConfigParameter(ThreadPoolConfigEntity threadPoolConfigEntity) {
        String cacheKey = RegistryEnumVO.THREAD_POOL_CONFIG_PARAMETER_LIST_KEY.getKey() + "_" + threadPoolConfigEntity.getAppName() + "_" + threadPoolConfigEntity.getThreadPoolName();
        RBucket<ThreadPoolConfigEntity> bucket = redissonClient.getBucket(cacheKey);
        bucket.set(threadPoolConfigEntity, Duration.ofDays(30));
    }
}

创建DynamicThreadPoolService类，用于管理线程池，包括获取线程池的状态，调整线程池的配置等。

// 动态线程池服务
public interface IDynamicThreadPoolService {

    // 查询全部的线程池
    List<ThreadPoolConfigEntity> queryThreadPoolList();

    // 根据名字查询线程池
    ThreadPoolConfigEntity queryThreadPoolConfigByName(String threadPoolName);

    // 更新线程池配置
    void updateThreadPoolConfig(ThreadPoolConfigEntity threadPoolConfigEntity);
}

具体实现类

public class DynamicThreadPoolService implements IDynamicThreadPoolService {

    private final Logger logger = LoggerFactory.getLogger(DynamicThreadPoolService.class);

    private final String applicationName;

    private final Map<String, ThreadPoolExecutor> threadPoolExecutorMap;

    public DynamicThreadPoolService(String applicationName, Map<String, ThreadPoolExecutor> threadPoolExecutorMap) {
        this.applicationName = applicationName;
        this.threadPoolExecutorMap = threadPoolExecutorMap;
    }

    @Override
    public List<ThreadPoolConfigEntity> queryThreadPoolList() {
        Set<String> threadPoolBeanNames = threadPoolExecutorMap.keySet();
        List<ThreadPoolConfigEntity> threadPoolVOS = new ArrayList<>(threadPoolBeanNames.size());
        for (String beanName : threadPoolBeanNames) {
            ThreadPoolExecutor executor = threadPoolExecutorMap.get(beanName);
            ThreadPoolConfigEntity threadPoolConfigVO = new ThreadPoolConfigEntity(applicationName,beanName);

            threadPoolConfigVO.setPoolSize(executor.getPoolSize());
            threadPoolConfigVO.setCorePoolSize(executor.getCorePoolSize());
            threadPoolConfigVO.setMaximumPoolSize(executor.getMaximumPoolSize());
            threadPoolConfigVO.setActiveCount(executor.getActiveCount());
            threadPoolConfigVO.setQueueType(executor.getQueue().getClass().getSimpleName());
            threadPoolConfigVO.setQueueSize(executor.getQueue().size());
            threadPoolConfigVO.setRemainingCapacity(executor.getQueue().remainingCapacity());

            threadPoolVOS.add(threadPoolConfigVO);
        }
        return threadPoolVOS;
    }

    @Override
    public ThreadPoolConfigEntity queryThreadPoolConfigByName(String threadPoolName) {
        ThreadPoolExecutor executor = threadPoolExecutorMap.get(threadPoolName);
        if(null == executor) return new ThreadPoolConfigEntity(applicationName,threadPoolName);

        ThreadPoolConfigEntity threadPoolConfigVO = new ThreadPoolConfigEntity(applicationName,threadPoolName);
        threadPoolConfigVO.setPoolSize(executor.getPoolSize());
        threadPoolConfigVO.setCorePoolSize(executor.getCorePoolSize());
        threadPoolConfigVO.setMaximumPoolSize(executor.getMaximumPoolSize());
        threadPoolConfigVO.setActiveCount(executor.getActiveCount());
        threadPoolConfigVO.setQueueType(executor.getQueue().getClass().getSimpleName());
        threadPoolConfigVO.setQueueSize(executor.getQueue().size());
        threadPoolConfigVO.setRemainingCapacity(executor.getQueue().remainingCapacity());

        return threadPoolConfigVO;
    }

    @Override
    public void updateThreadPoolConfig(ThreadPoolConfigEntity threadPoolConfigEntity) {
        if(null == threadPoolConfigEntity || !applicationName.equals(threadPoolConfigEntity.getAppName())) return;
        ThreadPoolExecutor executor = threadPoolExecutorMap.get(threadPoolConfigEntity.getThreadPoolName());
        if(null == executor) return;

        // 设置参数
        executor.setCorePoolSize(threadPoolConfigEntity.getCorePoolSize());
        executor.setMaximumPoolSize(threadPoolConfigEntity.getMaximumPoolSize());
    }
}

创建ThreadPoolDataReportJob类，用于定期向注册中心报告线程池的状态；创建ThreadPoolConfigAdjustListener类，用于监听Redis主题上的消息，当收到消息时，调整线程池的配置。

public class ThreadPoolDataReportJob {
    private Logger logger = LoggerFactory.getLogger(ThreadPoolDataReportJob.class);

    private final IDynamicThreadPoolService dynamicThreadPoolService;

    private final IRegistry registry;

    public ThreadPoolDataReportJob(IDynamicThreadPoolService dynamicThreadPoolService, IRegistry registry) {
        this.dynamicThreadPoolService = dynamicThreadPoolService;
        this.registry = registry;
    }

    @Scheduled(cron = "0/20 * * * * ?")
    public void execReportThreadPoolList(){
        List<ThreadPoolConfigEntity> poolList = dynamicThreadPoolService.queryThreadPoolList();
        registry.reportThreadPool(poolList);
        logger.info("动态线程池 上报线程池信息: {}", JSON.toJSONString(poolList));

        for (ThreadPoolConfigEntity threadPoolConfig : poolList) {
            registry.reportThreadPoolConfigParameter(threadPoolConfig);
            logger.info("动态线程池 上报线程池配置: {}", JSON.toJSONString(threadPoolConfig));
        }
    }
}

启动自动配置 resources/META-INF/spring.factories

@EnableAutoConfiguration 的作用是从 classpath 中搜索所有 META-INF/spring.factories 配置文件然后将其中 org.springframework.boot.autoconfigure.EnableAutoConfiguration 配置的信息加载到 spring 容器。

具体使用步骤

在项目的POM文件中引入该组件
在application.yml配置文件中配置redis注册中心的地址

dynamic:
  thread:
    pool:
      config:
        # 状态；true = 开启、false 关闭
        enabled: true
        # redis host
        host: 127.0.0.1
        # redis port
        port: 6379

创建ThreadExecutor的bean

@Bean("threadPoolExecutor01")
    public ThreadPoolExecutor threadPoolExecutor01(ThreadPoolConfigProperties properties) {
        RejectedExecutionHandler handler;
        switch (properties.getPolicy()){
            case "AbortPolicy":
                handler = new ThreadPoolExecutor.AbortPolicy();
                break;
            case "DiscardPolicy":
                handler = new ThreadPoolExecutor.DiscardPolicy();
                break;
            case "DiscardOldestPolicy":
                handler = new ThreadPoolExecutor.DiscardOldestPolicy();
                break;
            case "CallerRunsPolicy":
                handler = new ThreadPoolExecutor.CallerRunsPolicy();
                break;
            default:
                handler = new ThreadPoolExecutor.AbortPolicy();
                break;
        }
        // 创建线程池
        return new ThreadPoolExecutor(properties.getCorePoolSize(),
                properties.getMaxPoolSize(),
                properties.getKeepAliveTime(),
                TimeUnit.SECONDS,
                new LinkedBlockingQueue<>(properties.getBlockQueueSize()),
                Executors.defaultThreadFactory(),
                handler);
    }