forkJoin集成spring boot

最新推荐文章于 2025-04-18 15:00:00 发布

原创最新推荐文章于 2025-04-18 15:00:00 发布 · 1.6k 阅读

3 ·

CC 4.0 BY-SA版权

文章标签：

#java #spring boot #intellij-idea #forkjoin

本文介绍如何使用Spring Boot集成ForkJoin框架，实现对大量JobModel的高效并发处理，通过分批和并行化策略提高数据表创建的性能。作者通过实例展示了如何创建任务、划分工作负载以及监控执行过程。

forkJoin集成spring boot处理大量数据添加

import com.wugui.datax.admin.entity.JobModel;
import com.wugui.datax.admin.entity.JobModelLog;
import com.wugui.datax.admin.service.DatasourceQueryService;
import com.wugui.datax.admin.service.JobModelLogService;
import com.wugui.datax.admin.service.JobModelService;
import lombok.SneakyThrows;
import org.springframework.beans.factory.annotation.Autowired;

import java.io.IOException;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.RecursiveAction;

/**
 * @Author Mr.zhang
 * @Date: 2022/02/17/ 9:50
 * @Description
 */
public class BatchTask extends RecursiveAction {

    // 临界值
    private static final int THRESHOLD = 10;

    private List<JobModel> list;


    private JobModelService jobModelService;

    private DatasourceQueryService datasourceQueryService;

    private JobModelLogService jobModelLogService;

    public BatchTask(List<JobModel> list,DatasourceQueryService datasourceQueryService,JobModelService jobModelService,JobModelLogService jobModelLogService) {
        this.jobModelService = jobModelService;
        this.datasourceQueryService = datasourceQueryService;
        this.jobModelLogService = jobModelLogService;
        this.list = list;
    }

    @SneakyThrows
    @Override
    protected void compute() {
        boolean compute = list.size() <= THRESHOLD;
        if (compute) {
            for (JobModel s : list) {
                String table = datasourceQueryService.createTable(s.getDatasourseId(),s.getCreateTableSql());
                JobModel byId = jobModelService.getById(s.getId());
                JobModelLog jobModelLog = new JobModelLog();
                jobModelLog.setModelId(byId.getId());
                jobModelLog.setModelName(byId.getModelName());
                jobModelLog.setProjectId(byId.getProjectId());
                if (table.equals("false")){
                    jobModelLog.setStare("成功");
                    jobModelLog.setExecuteLog("成功");
                    byId.setStart(2);
                    jobModelService.updateById(byId);
                }else {
                    jobModelLog.setStare("失败");
                    jobModelLog.setExecuteLog(table);
                }
                jobModelLogService.save(jobModelLog);
            }
        } else {
            List<List<JobModel>> lists = BatchTask.averageAssign(list, 2);
            // 递归
            BatchTask task1 = new BatchTask(lists.get(0), datasourceQueryService,jobModelService,jobModelLogService);
            BatchTask task2 = new BatchTask(lists.get(1), datasourceQueryService,jobModelService,jobModelLogService);
            // 拆分任务，把任务压入线程队列
            invokeAll(task1, task2);
        }
    }


    /**
     * 将一组数据平均分成n组
     *
     * @param source 要分组的数据源
     * @param n      平均分成n组
     * @param <T>
     * @return
     */
    public static <T> List<List<T>> averageAssign(List<T> source, int n) {
        List<List<T>> result = new ArrayList<>();
        int remainder = source.size() % n;  //(先计算出余数)
        int number = source.size() / n;  //然后是商
        int offset = 0;//偏移量
        for (int i = 0; i < n; i++) {
            List<T> value;
            if (remainder > 0) {
                value = source.subList(i * number + offset, (i + 1) * number + offset + 1);
                remainder--;
                offset++;
            } else {
                value = source.subList(i * number + offset, (i + 1) * number + offset);
            }
            result.add(value);
        }
        return result;
    }

}

调用案例

  @PostMapping("createTableList")
    @ApiOperation(value = "批量执行模型")
    public ReturnT<String> createTableList(@RequestBody List<JobModel> list) {
        System.err.println("=======开始========");
        ForkJoinPool pool = null;
        try {
            pool = new ForkJoinPool(10);
            BatchTask task = new BatchTask(list, datasourceQueryService, jobModelService,jobModelLogService);
            ForkJoinTask<Void> submit = pool.submit(task);
            submit.get();
        } catch (InterruptedException e) {
            e.printStackTrace();
        } catch (ExecutionException e) {
            e.printStackTrace();
        } finally {
            // 关闭线程池
            pool.shutdown();
        }
        System.err.println("=======结束========");
        return new ReturnT<>("执行完成");
    }