ThreadPoolTaskExecutor多线程跑任务时数据未跑完

最新推荐文章于 2024-04-19 10:04:46 发布

色空大师

最新推荐文章于 2024-04-19 10:04:46 发布

阅读量261

点赞数

文章标签：开发语言 java 线程池

本文链接：https://blog.csdn.net/weixin_44860141/article/details/133883560

版权

1.问题描述：循环查数据，然后用多线程去更新查到的数据

代码如下：

建表语句

CREATE TABLE `tb_user` (
  `id` int NOT NULL AUTO_INCREMENT COMMENT 'id',
  `name` varchar(50) DEFAULT NULL COMMENT '名称',
  PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4

实体类

@Data
@TableName("tb_user")
public class TbUser {

    @TableId(type = IdType.AUTO)
    private int id;

    @TableField("name")
    private String name;
}

mapper

@Mapper
@Repository
public interface TbUserMapper extends BaseMapper<TbUser> {

    int updateByIds(List<TbUser> list);
}

xml文件

<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
        "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.test.netty.mapper.TbUserMapper">

    <update id="updateByIds">

       <foreach collection="list" item="user" separator=";">
           update tb_user set name = #{user.name} where id = #{user.id}
       </foreach>
    </update>
</mapper>

service

package com.test.netty.service;

import com.baomidou.mybatisplus.extension.service.IService;
import com.test.netty.pojo.TbUser;

public interface UserService extends IService<TbUser> {
    void testThread();
}

实现类

package com.test.netty.service.impl;

import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.test.netty.mapper.TbUserMapper;
import com.test.netty.pojo.TbUser;
import com.test.netty.service.UserService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import org.springframework.stereotype.Service;

import javax.annotation.Resource;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;

import static com.baomidou.mybatisplus.extension.toolkit.Db.updateBatchById;

@Service
@Slf4j
public class UserServiceImpl extends ServiceImpl<TbUserMapper, TbUser> implements UserService {
    @Resource
    private TbUserMapper userMapper;
    
    @Autowired
    @Qualifier("taskExecutor")
    private ThreadPoolTaskExecutor taskExecutor;

    @Override
    public void testThread() {

        log.info("开始批量更新流程");
        long start = System.currentTimeMillis();
        Long count = userMapper.selectCount(null);
        Integer pageSize = 500;
        int totalPages = count.intValue()/pageSize;
        if (count.intValue() % pageSize ==0){
            totalPages ++;
        }

        for (int i=1;i<totalPages;i++){
            Page<TbUser> page = userMapper.selectPage(new Page<TbUser>(i, 500), null);
            List<TbUser> list = page.getRecords();
            List<TbUser> users = new ArrayList<>();

            list.stream().forEach(user->{
                user.setName("test6");
                users.add(user);
            });
            int finalI = i;
            taskExecutor.execute(new Runnable() {
                @Override
                public void run() {
                    log.info("第{}次更新", finalI);
                    updateBatchById(users,500);
                    //userMapper.updateByIds(users);
                    log.info("第{}次更新完成", finalI);
                }
            });
        }
        

        log.info("所有数据更新完成，耗时：{}",System.currentTimeMillis()-start);
    }
}

线程池配置

package com.test.netty.config;

import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;

import java.util.Currency;
import java.util.concurrent.ThreadPoolExecutor;

@Configuration
@Slf4j
public class TaskExecutorPoolConfig {

    private static final int workQueue = 5000;

    private static final int keepActiveTime = 30;

    private static final int CPU_COUNT = Runtime.getRuntime().availableProcessors();

    private static final int corePoolSize = 2 * CPU_COUNT;

    private static final int maxPoolSize = CPU_COUNT * 5;

    @Bean("taskExecutor")
    public ThreadPoolTaskExecutor myTaskExecutor(){
        ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
        executor.setThreadNamePrefix("poolTest-");
        executor.setCorePoolSize(corePoolSize);
        executor.setMaxPoolSize(maxPoolSize);
        executor.setQueueCapacity(workQueue);
        executor.setKeepAliveSeconds(keepActiveTime);
        executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
        executor.initialize();
        return executor;
    }
}

mybatisPlus分页配置

package com.test.netty.config;

import com.baomidou.mybatisplus.annotation.DbType;
import com.baomidou.mybatisplus.extension.plugins.MybatisPlusInterceptor;
import com.baomidou.mybatisplus.extension.plugins.inner.PaginationInnerInterceptor;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

@Configuration
public class MybatisConfig {

    @Bean
    public MybatisPlusInterceptor mybatisPlusInterceptor(){
        MybatisPlusInterceptor interceptor = new MybatisPlusInterceptor();
        interceptor.addInnerInterceptor(new PaginationInnerInterceptor());
        return interceptor;
    }
}

测试类

单元测试需要添加test依赖

<dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-test</artifactId>
        </dependency>

package com.test.netty.controller;

import com.test.netty.mapper.DataMapper;
import com.test.netty.mapper.TbUserMapper;
import com.test.netty.pojo.DataDto;
import com.test.netty.pojo.TbUser;
import com.test.netty.service.DataService;
import com.test.netty.service.TestService;
import com.test.netty.service.UserService;
import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;

import java.util.ArrayList;
import java.util.List;

import static org.junit.jupiter.api.Assertions.*;
@SpringBootTest
@Slf4j
class TestControllerTest {


    @Autowired
    private TbUserMapper userMapper;

    @Autowired
    private UserService userService;

    @Test
    void test1() {
    }

    @Test
    public void insertBatch(){
        for (int i=1;i<=10000;i++){
            TbUser user = new TbUser();
            user.setName(String.valueOf(i));
            userMapper.insert(user);
            log.info("第{}次插入",i);
        }
    }

    @Test
    public void test(){
        userService.testThread();
    }







}

application配置文件

server:
  port: 8081
spring:
  application:
    name: netty-test-01
  datasource:
    driver-class-name: com.mysql.cj.jdbc.Driver
    url: jdbc:mysql://localhost:3306/test?serverTimezone=Asia/Shanghai&characterEncoding=utf-8&allowMultiQueries=true
    username: root
    password: root

步骤：

1.先运行测试类insertBatch方法，插入数据10000条。
2.运行test方法进行更新

问题

发现执行完成后更新的数据不完整，只有五千左右
在这里插入图片描述
然后自己写sql：
将实现类的方法注解放开

运行后发现数据全部更新完成

注意：test7或test8是我每次运行都会改一下修改的值，这个随便写，不用在意。

解决方法：使用ExecutorService线程池

未找到出现这个问题的原因，但是换了一个线程池后问题解决
在实现类中创建线程池
在这里插入图片描述

完成代码如下：

package com.test.netty.service.impl;

import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.test.netty.mapper.TbUserMapper;
import com.test.netty.pojo.TbUser;
import com.test.netty.service.UserService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import org.springframework.stereotype.Service;

import javax.annotation.Resource;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;

import static com.baomidou.mybatisplus.extension.toolkit.Db.updateBatchById;

@Service
@Slf4j
public class UserServiceImpl extends ServiceImpl<TbUserMapper, TbUser> implements UserService {
    @Resource
    private TbUserMapper userMapper;

    @Autowired
    @Qualifier("taskExecutor")
    private ThreadPoolTaskExecutor taskExecutor;

    @Override
    public void testThread() {
        ExecutorService executorService = Executors.newFixedThreadPool(30);

        log.info("开始批量更新流程");
        long start = System.currentTimeMillis();
        Long count = userMapper.selectCount(null);
        Integer pageSize = 500;
        int totalPages = count.intValue()/pageSize;
        if (count.intValue() % pageSize ==0){
            totalPages ++;
        }

        for (int i=1;i<totalPages;i++){
            Page<TbUser> page = userMapper.selectPage(new Page<TbUser>(i, 500), null);
            List<TbUser> list = page.getRecords();
            List<TbUser> users = new ArrayList<>();

            list.stream().forEach(user->{
                user.setName("test6");
                users.add(user);
            });
            int finalI = i;
            executorService.execute(new Runnable() {
                @Override
                public void run() {
                    log.info("第{}次更新", finalI);
                    updateBatchById(users,500);
                    userMapper.updateByIds(users);
                    log.info("第{}次更新完成", finalI);
                }
            });
        }
        executorService.shutdown();
        try {
            executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS);
        }catch (Exception e){
            log.error("error");
        }

        log.info("所有数据更新完成，耗时：{}",System.currentTimeMillis()-start);
    }
}