【SpringBoot并发事务处理】

写程序的小熊猫

已于 2024-03-12 18:06:50 修改

阅读量469

点赞数 9

文章标签： spring boot 后端 java

于 2024-03-12 18:06:28 首次发布

本文链接：https://blog.csdn.net/qq614571540/article/details/136655046

版权

需求

假设现在有1000个项目，要从excel文件导入系统，即有一个1000行的文件导入系统，因为导入的时候要对每个项目做校验，导致导入很慢，现在需要进行优化。
规则：

如果有项目名称重复，则提示项目重复。
如果以前数据库没有此名称项目则新增，如果有则修改。

涉及技术

EasyExcel，SpringBoot线程池，并发异步处理

思路

这个需求耗时有两点，1是进行校验，2是进行添加修改操作
针对第一点 ：我们使用CompletableFuture进行异步校验；
针对第二点 ：为了保证事务，我们使用两个CountDownLatch和AtomicBoolean保证事务，mainLatch 和rollbackLatch用来阻塞事务提交，使用isSuccess来代表事务是否提交

细节点

同一个类中异步注解失效，需要重新获取bean对象调用方法才可以

实例

ProjectServiceImpl

package com.qp.projectJUC.service.impl;

import cn.hutool.core.lang.UUID;
import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.exception.ExcelDataConvertException;
import com.google.common.collect.Lists;
import com.qp.projectJUC.listener.ProjectImportListener;
import com.qp.projectJUC.mapper.ProjectInfoMapper;
import com.qp.projectJUC.pojo.ProjectImportDto;
import com.qp.projectJUC.pojo.ProjectInfo;
import com.qp.projectJUC.service.ProjectInfoService;
import com.qp.projectJUC.util.SpringContextUtil;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.scheduling.annotation.Async;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.interceptor.TransactionAspectSupport;
import org.springframework.web.multipart.MultipartFile;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.Collectors;


@Slf4j
@Service
public class ProjectInfoServiceImpl implements ProjectInfoService {

    @Autowired
    @Qualifier("projectImportExecutor")
    private ThreadPoolTaskExecutor projectImportExecutor;

    @Autowired
    private ProjectInfoMapper projectInfoMapper;

    @SuppressWarnings("rawtypes")
    @Override
    @Transactional(rollbackFor = Exception.class)
    public List<String> importProject(MultipartFile file) {
        String seq = UUID.fastUUID().toString();
        String msg = "ok";
        List<ProjectImportDto> parsedData = new CopyOnWriteArrayList<>();
        ProjectImportListener listener = new ProjectImportListener(parsedData);
        try {
            EasyExcel.read(file.getInputStream(), ProjectImportDto.class, listener).sheet().headRowNumber(1).doRead();
            CompletableFuture[] futures = Lists.partition(parsedData, 30).stream().map(batch -> CompletableFuture.runAsync(() -> listener.new PreCheck(batch, seq).preCheck(), projectImportExecutor).whenComplete((res, ex) -> {
                if (ex != null) {
                    listener.getSuccess().set(false);
                    log.error("seq=>{},import target value fail, e:", seq, ex);
                }
            })).toArray(CompletableFuture[]::new);
            CompletableFuture.allOf(futures).get();

            //校验失败返回信息
            if (!listener.getSuccess().get()) {
                return listener.getMsg();
            }
            //校验重名
            List<String> sameName = parsedData.stream().map(ProjectImportDto::getProjectName).collect(Collectors.toMap(e -> e, e -> 1, Integer::sum)).entrySet().stream().filter(entry -> entry.getValue() > 1).map(Map.Entry::getKey).collect(Collectors.toList());
            if (CollectionUtils.isNotEmpty(sameName)) {
                List<String> s = new ArrayList<>();
                sameName.forEach(name -> s.add("【" + name + "】" + "项目名重名"));
                return s;
            }

            //进行入库
            List<ProjectImportDto> createList = new CopyOnWriteArrayList<>();
            List<ProjectImportDto> updateList = new CopyOnWriteArrayList<>();
            for (ProjectImportDto parsedDatum : parsedData) {
                ProjectInfo po = parsedDatum.getProjectInfo();
                //查询状态
                List<ProjectInfo> projectInfoList = projectInfoMapper.selectByName(po.getProjectName());
                if (CollectionUtils.isEmpty(projectInfoList)) {
                    createList.add(parsedDatum);
                } else {
                    updateList.add(parsedDatum);
                }
            }
            List<List<ProjectImportDto>> createTasks = Lists.partition(createList, 30);
            List<List<ProjectImportDto>> updateTasks = Lists.partition(updateList, 30);
            AtomicBoolean isSuccess = new AtomicBoolean(true);
            CountDownLatch mainLatch = new CountDownLatch(createTasks.size() + updateTasks.size());
            CountDownLatch rollbackLatch = new CountDownLatch(1);

            ProjectInfoServiceImpl service = SpringContextUtil.getBean(ProjectInfoServiceImpl.class);

            for (List<ProjectImportDto> part : createTasks) {
                service.createProject(part, mainLatch, rollbackLatch, isSuccess);
            }

            for (List<ProjectImportDto> part : updateTasks) {
                //更新
                service.updateProject(part, mainLatch, rollbackLatch, isSuccess);
            }
            try {
                mainLatch.await();
            } finally {
                rollbackLatch.countDown();
            }
            if (!isSuccess.get()) {
                return Lists.newArrayList("项目导入错误");
            }
        } catch (IOException e) {
            log.error("seq=>{},import target value fail, e:", seq, e);
            throw new RuntimeException("file_error");
        } catch (ExcelDataConvertException e) {
            log.error("seq=>{},项目导入模板不正确", seq, e);
            return Lists.newArrayList("项目导入模板不正确");
        } catch (Exception e) {
            log.error("seq=>{},项目导入错误", seq, e);
            return Lists.newArrayList("项目导入错误");
        }
        return Lists.newArrayList(msg);
    }

    @Async("projectImportExecutor")
    @Transactional(rollbackFor = Exception.class)
    public void createProject(List<ProjectImportDto> datas, CountDownLatch mainLatch, CountDownLatch rollbackLatch, AtomicBoolean isSuccess) {
        try {
            for (ProjectImportDto data : datas) {
                if (!isSuccess.get()) {
                    throw new RuntimeException("file_error");
                }
                //创建项目方法
//                createImportProject(data.getProjectInfo());
            }
        } catch (Exception e) {
            isSuccess.set(false);
            log.error("import project fail, e:", e);
        } finally {
            mainLatch.countDown();
            try {
                rollbackLatch.await();
                if (!isSuccess.get()) {
                    //事务回滚
                    TransactionAspectSupport.currentTransactionStatus().setRollbackOnly();
                }
            } catch (InterruptedException e) {
                isSuccess.set(false);
                log.error("Create project code fail", e);
                //事务回滚
                TransactionAspectSupport.currentTransactionStatus().setRollbackOnly();
            }
        }

    }

    @Async("projectImportExecutor")
    @Transactional(rollbackFor = Exception.class)
    public void updateProject(List<ProjectImportDto> datas, CountDownLatch mainLatch, CountDownLatch rollbackLatch, AtomicBoolean isSuccess) {
        try {
            for (ProjectImportDto data : datas) {
                if (!isSuccess.get()) {
                    throw new RuntimeException("file_error");
                }
                //更新项目方法
//                updateImportProject(data.getProjectInfo());
            }
        } catch (Exception e) {
            isSuccess.set(false);
            log.error("import project fail, e:", e);
        } finally {
            mainLatch.countDown();
            try {
                rollbackLatch.await();
                if (!isSuccess.get()) {
                    //事务回滚
                    TransactionAspectSupport.currentTransactionStatus().setRollbackOnly();
                }
            } catch (InterruptedException e) {
                isSuccess.set(false);
                log.error("update import project code fail", e);
                //事务回滚
                TransactionAspectSupport.currentTransactionStatus().setRollbackOnly();
            }
        }
    }
}

listener

package com.qp.projectJUC.listener;


import com.alibaba.excel.context.AnalysisContext;
import com.alibaba.excel.read.listener.ReadListener;
import com.qp.projectJUC.pojo.ProjectImportDto;
import com.qp.projectJUC.pojo.ProjectInfo;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;

import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;

@Slf4j
@Data
public class ProjectImportListener implements ReadListener<ProjectImportDto> {

    private List<ProjectImportDto> results;

    private List<ProjectInfo> projectInfoList;
    /**
     * 返回信息
     */
    public List<String> msg;
    /**
     * 是否解析成功
     */
    public AtomicBoolean success;

    public ProjectImportListener(List<ProjectImportDto> results) {
        this.results = results;
    }


    @Override
    public void invoke(ProjectImportDto data, AnalysisContext context) {
        log.info("项目导入start");
        data.setRowIndex(context.readRowHolder().getRowIndex());
        results.add(data);
    }

    @Override
    public void doAfterAllAnalysed(AnalysisContext context) {
        //do nothing
    }

    public class PreCheck {

        private String seq;

        private List<ProjectImportDto> data;

        public PreCheck(List<ProjectImportDto> data, String seq) {
            this.data = data;
            this.seq = seq;
        }

        public void preCheck() {
            log.info("start pre check seq:{}", seq);

            for (ProjectImportDto dto : data) {
                Integer rowIndex = dto.getRowIndex();
                int rowNum = rowIndex + 1;
                //模板校验
                if (rowIndex < 1) {
                    checkModule(dto, rowIndex);
                    continue;
                }
                if (!success.get()) {
                    return;
                }

                ProjectInfo projectInfo = new ProjectInfo();
                //进行校验，并且给ProjectInfo赋值
                if (StringUtils.isBlank(dto.getProjectName())) {
                    success.set(false);
                    msg.add("第" + rowNum + "行数据【项目名称】字段不正确");
                }else{
                    projectInfo.setProjectName(dto.getProjectName());
                }
                if (StringUtils.isBlank(dto.getLeaderName())) {
                    success.set(false);
                    msg.add("第" + rowNum + "行数据【项目主人】字段不正确");
                }else{
                    projectInfo.setLeaderName(dto.getLeaderName());
                }
                if (StringUtils.isBlank(dto.getRemark())) {
                    success.set(false);
                    msg.add("第" + rowNum + "行数据【项目描述】字段不正确");
                }else{
                    projectInfo.setRemark(dto.getRemark());
                }
                dto.setProjectInfo(projectInfo);
            }
        }

        public void checkModule(ProjectImportDto dto, Integer rowIndex) {
            if (!StringUtils.equals("* 序号", dto.getIndex())) {
                success.set(false);
            }

            if (!StringUtils.equals("* 项目名称", dto.getProjectName())) {
                success.set(false);
            }
            if (!StringUtils.equals("* 项目主人姓名", dto.getLeaderName())) {
                success.set(false);
            }
            if (!StringUtils.equals("* 项目描述", dto.getRemark())) {
                success.set(false);
            }
        }
    }
}

ProjectImportDto

package com.qp.projectJUC.pojo;


import com.alibaba.excel.annotation.ExcelProperty;
import lombok.Data;

@Data
public class ProjectImportDto {

    @ExcelProperty(value = "序号", index = 0)
    private String index;
    @ExcelProperty(value = "项目名称", index = 1)
    private String projectName;
    @ExcelProperty(value = "项目主人名称", index = 2)
    private String leaderName;
    @ExcelProperty(value = "项目描述", index = 3)
    private String remark;

    private ProjectInfo projectInfo;
    private Integer rowIndex;
}

ProjectInfo

package com.qp.projectJUC.pojo;

import lombok.Data;

@Data
public class ProjectInfo {

    private String projectName;
    private String leaderName;
    private String remark;
}

SpringContextUtil

package com.qp.projectJUC.util;


import org.springframework.beans.BeansException;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
import org.springframework.stereotype.Component;

@Component
public class SpringContextUtil implements ApplicationContextAware {

    private static volatile ApplicationContext applicationContext;

    @Override
    public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
        SpringContextUtil.applicationContext = applicationContext;
    }

    /**
     * 获取ApplicationContext
     * @return
     */
    public static ApplicationContext getApplicationContext() {
        return applicationContext;
    }

    /**
     * 通过class获取Bean
     * @param clazz
     * @param <T>
     * @return
     */
    public static <T> T getBean(Class<T> clazz) {
        return applicationContext.getBean(clazz);
    }

    /**
     * 通过name以及class获取Bean
     * @param name
     * @param clazz
     * @param <T>
     * @return
     */
    public static <T> T getBean(String name, Class<T> clazz) {
        return applicationContext.getBean(name, clazz);
    }
}

线程池

package com.qp.projectJUC.config;


import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;

import java.util.concurrent.ThreadPoolExecutor;

@Configuration
public class ExecutorConfig {

    @Bean(name = "projectImportExecutor")
    public ThreadPoolTaskExecutor projectImportExecutor() {
        ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
        int cpuNum = Runtime.getRuntime().availableProcessors();
        // 设置核心线程数
        executor.setCorePoolSize(20);
        // 设置最大线程数
        executor.setMaxPoolSize(30);
        // 设置队列容量
        executor.setQueueCapacity(1000);
        // 设置默认线程名称
        executor.setThreadNamePrefix("project-import-");
        // 设置拒绝策略
        executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
        // 等待所有任务结束后再关闭线程池. 当线程池要关闭的时候用  ExecutorService.shutdown(), ExecutorService.shutdownNow()
        // ExecutorService.shutdown(); 停止接收新任务，原来的任务继续执行
        // ExecutorService.shutdownNow();停止接收新任务，原来的任务停止执行
        executor.setWaitForTasksToCompleteOnShutdown(true);
        //设置线程池中任务的等待时间，如果超过这个时候还没有销毁就强制销毁，以确保能够被关闭，而不是阻塞住
        executor.setAwaitTerminationSeconds(60);
        executor.setAllowCoreThreadTimeOut(true);
        return executor;
    }
}

maven依赖

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>
    <parent>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-parent</artifactId>
        <version>2.6.15</version>
        <relativePath/>
    </parent>

    <groupId>com.qp</groupId>
    <artifactId>study</artifactId>
    <version>1.0.0</version>

    <name>study</name>

    <properties>
        <java.version>8</java.version>
        <mysql.version>8.0.33</mysql.version>
        <fastjson.version>2.0.38</fastjson.version>
        <spring-cloud.version>2021.0.8</spring-cloud.version>
        <hutool.version>5.8.12</hutool.version>
    </properties>

    <dependencyManagement>
        <dependencies>
            <dependency>
                <groupId>org.springframework.cloud</groupId>
                <artifactId>spring-cloud-dependencies</artifactId>
                <version>${spring-cloud.version}</version>
                <type>pom</type>
                <scope>import</scope>
            </dependency>
        </dependencies>
    </dependencyManagement>

    <dependencies>
        <!-- SpringBoot相关 -->
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter</artifactId>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
        </dependency>


        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>fastjson</artifactId>
            <version>1.2.83</version>
        </dependency>

        <!--mysql 数据库相关-->
        <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <version>${mysql.version}</version>
        </dependency>
        <dependency>
            <groupId>com.zaxxer</groupId>
            <artifactId>HikariCP</artifactId>
        </dependency>
        <dependency>
            <groupId>com.github.pagehelper</groupId>
            <artifactId>pagehelper-spring-boot-starter</artifactId>
            <version>1.4.5</version>
        </dependency>

        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
            <scope>compile</scope>
        </dependency>

        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-test</artifactId>
            <scope>test</scope>
        </dependency>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.13.2</version>
            <scope>test</scope>
        </dependency>

        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>easyexcel</artifactId>
            <version>3.2.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.commons</groupId>
            <artifactId>commons-lang3</artifactId>
            <version>3.12.0</version>
        </dependency>
        <dependency>
            <groupId>com.google.guava</groupId>
            <artifactId>guava</artifactId>
            <version>32.1.3-jre</version>
        </dependency>
        <dependency>
            <groupId>cn.hutool</groupId>
            <artifactId>hutool-all</artifactId>
            <version>${hutool.version}</version>
        </dependency>
    </dependencies>

    <build>
        <finalName>study</finalName>
        <plugins>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-compiler-plugin</artifactId>
                <version>3.8.1</version>
                <configuration>
                    <encoding>UTF-8</encoding>
                    <source>${java.version}</source>
                    <target>${java.version}</target>
                </configuration>
            </plugin>
        </plugins>
    </build>
</project>