使用线程池 插入万条数据做压力测试

实习遇到压力测试自己写一个程序做压力测试 ，使用线程池教程

1.在yml文件中添加一个线程池的配置

# 线程池基础参数
task:
  pool:
    corePoolSize: 8 # 核心线程数
    maxPoolSize: 20 # 设置最大线程数
    keepAliveSeconds: 300 # 设置线程活跃时间
    queueCapacity: 9999 # 设置队列容量

 2在Spring中配置一个Bean,作为线程池的 配置bean 将yml中的参数拿出来

//设置yml中的前缀 task.pool

@ConfigurationProperties(prefix = "task.pool")
public class TaskThreadInfo {
    private Integer corePoolSize;
    private Integer maxPoolSize;
    private Integer keepAliveSeconds;
    private Integer queueCapacity;

    public Integer getCorePoolSize() {
        return corePoolSize;
    }

    public void setCorePoolSize(Integer corePoolSize) {
        this.corePoolSize = corePoolSize;
    }

    public Integer getMaxPoolSize() {
        return maxPoolSize;
    }

    public void setMaxPoolSize(Integer maxPoolSize) {
        this.maxPoolSize = maxPoolSize;
    }

    public Integer getKeepAliveSeconds() {
        return keepAliveSeconds;
    }

    public void setKeepAliveSeconds(Integer keepAliveSeconds) {
        this.keepAliveSeconds = keepAliveSeconds;
    }

    public Integer getQueueCapacity() {
        return queueCapacity;
    }

    public void setQueueCapacity(Integer queueCapacity) {
        this.queueCapacity = queueCapacity;
    }

}

 3.在新建一个配置TaskExecutePoolConfig.java类，将ThreadPoolTaskExecutor 实现，具体代码如下

@Configuration
@EnableConfigurationProperties(TaskThreadInfo.class)
public class TaskExecutePoolConfig {
    private TaskThreadInfo taskThreadInfo;
    public TaskExecutePoolConfig(TaskThreadInfo info){
        this.taskThreadInfo = info;
    }
    /**
     * 线程池配置
     */
    @Bean(name = "threadPoolTaskExecutor",destroyMethod = "shutdown")
    public ThreadPoolTaskExecutor threadPoolTaskExecutor(){
        ThreadPoolTaskExecutor taskExecutor = new ThreadPoolTaskExecutor();
        //设置最大 的 核心线程数
        taskExecutor.setCorePoolSize(taskThreadInfo.getCorePoolSize());
        //设置最大线程数
        taskExecutor.setMaxPoolSize(taskThreadInfo.getMaxPoolSize());
        //设置 线程空闲时间
        taskExecutor.setKeepAliveSeconds(taskThreadInfo.getKeepAliveSeconds());
        //设置最大队列
        taskExecutor.setQueueCapacity(taskThreadInfo.getQueueCapacity());
        //设置拒绝策略
        taskExecutor.setRejectedExecutionHandler(rejectedExecutionHandler());
        //设置线程前缀名字
        taskExecutor.setThreadNamePrefix("多表插入-Async-");
        taskExecutor.initialize();
        return taskExecutor;
    }
    //自定义线程拒绝策略


    @Bean
    public  RejectedExecutionHandler rejectedExecutionHandler(){
        RejectedExecutionHandler errorHandler = new RejectedExecutionHandler() {
            @Override
            public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {
                System.out.println("线程池异常");
            }
        };
        return errorHandler;
    }

}

4实现方法

4.1 企业级开发往往设计多表联查，或者多表关联。所以在保存数据时需要使用到list集合。作为新手小白，我对于这个list同步问题可谓是伤透脑筋，

4.2使用 Collections.synchronizedList(new ArrayList<>());使用集合安全的list进行保存

List<RecUser> list = Collections.synchronizedList(new ArrayList<>());

4.3对其上锁Lock lock = new ReentrantLock();保存插入时不会丢失数据

 Lock lock = new ReentrantLock();

//上锁
lock.lock();
try {
  list.add(recUser);
} finally {
解锁
 lock.unlock();
 }

4.4创建CountDownLatch对象.通过使用CountDownLatch，主线程会等待所有线程执行完毕后再返回list，确保返回的list不为空。

CountDownLatch latch = new CountDownLatch(number); // 

 for (int count = 0; count < number; count++) {
        final int index = i;
        executor.execute(() -> {
            // 省略部分代码...

            lock.lock();
            try {
                list.add(recUser);
            } finally {
                lock.unlock();
            }
            log.info("当前线程池的数量为： {}", executor.getPoolSize());

            latch.countDown(); // 减少计数器的值
        });
    }

     try {
         latch.await();// 等待计数器变为0
        }catch (Exception e){
            e.getMessage();
        }
    log.info("执行线程成功");
    System.out.println("rec_user插入成功！");
    return list;
}

5.使用的总体完成效果

@Service
@Slf4j
public class RecUserServiceImpl extends ServiceImpl<RecUserMapper, RecUser> implements RecUserService {

    @Autowired
    private RecUserMapper recUserMapper;
    @Autowired
    private ThreadPoolTaskExecutor executor;

    public List<RecUser> saveIntoRecUser(int number){
        List<RecUser> list = Collections.synchronizedList(new ArrayList<>());
        Lock lock = new ReentrantLock();
        CountDownLatch latch = new CountDownLatch(number); // 创建CountDownLatch对象
        int i = 0 ;
        log.info("线程池初始化大小为{}",executor.getPoolSize());
        for (int count = 0 ; count < number ;count++){
            final int index = i;
             executor.execute(()->{
                    RecUser recUser = new RecUser();
                    recUser.setUserName("张三" + index);
                   
                    lock.lock();
                    try {
                        list.add(recUser);
                    } finally {
                        lock.unlock();
                    }
                    log.info("当前线程池的数量为： {}",executor.getPoolSize());
                    latch.countDown(); // 减少计数器的值
            });
        }

        try {
            latch.await();// 等待计数器变为0
        }catch (Exception e){
            e.getMessage();
        }
        log.info("执行线程成功");
        this.saveBatch(list);
        System.out.println("rec_user插入成功！");
        return list;
    }

}

6小tips 如果使用guava包中的Lists.partition方法对list切割会更快

         <dependency>
            <groupId>com.google.guava</groupId>
            <artifactId>guava</artifactId>
            <version>29.0-jre</version>
        </dependency>

     this.saveBatch(recA01s);



        Lists.partition(recA01s,20).forEach(item ->{
            this.saveBatch(item);
        });

7效果展示