使用forkjoin批量插入数据库表

forkjoin基本使用

链接: 多线程之Fork/Join使用.

模拟导入数据

import com.baomidou.mybatisplus.annotation.TableName;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import java.io.Serializable;
import lombok.Data;
import lombok.EqualsAndHashCode;

/**
 * <p>
 * 
 * </p>
 *
 * @author zhoum
 * @since 2022-01-30
 */
@Data
@EqualsAndHashCode(callSuper = false)
@TableName("user_test")
public class UserTestEntity implements Serializable {

    private static final long serialVersionUID = 1L;

    @TableId(value = "id", type = IdType.ASSIGN_ID)
    private String id;

    private String name1;

    private String name2;

    private String name3;

    private String name4;

    private String name5;

    private String name6;

    private String name7;

    private String name8;

    private String name9;


}

package com.zm.forkjoin;

import com.zm.entity.UserTestEntity;
import com.zm.mapper.UserTestMapper;

import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.RecursiveAction;

public class BatchTask extends RecursiveAction {

    // 临界值
    private static final int THRESHOLD = 10;

    private List<UserTestEntity> list;

    private UserTestMapper userTestMapper;

    public BatchTask(List<UserTestEntity> list, UserTestMapper userTestMapper) {
        this.userTestMapper = userTestMapper;
        this.list = list;
    }

    @Override
    protected void compute() {
        boolean compute = list.size() <= THRESHOLD;
        if (compute) {
            for (UserTestEntity userTestEntity : list) {
                userTestMapper.insert(userTestEntity);
            }
        } else {
            List<List<UserTestEntity>> lists = BatchTask.averageAssign(list, 2);
            // 递归
            BatchTask task1 = new BatchTask(lists.get(0), userTestMapper);
            BatchTask task2 = new BatchTask(lists.get(1), userTestMapper);
            // 拆分任务，把任务压入线程队列
            invokeAll(task1, task2);
        }
    }


    /**
     * 将一组数据平均分成n组
     *
     * @param source 要分组的数据源
     * @param n      平均分成n组
     * @param <T>
     * @return
     */
    public static <T> List<List<T>> averageAssign(List<T> source, int n) {
        List<List<T>> result = new ArrayList<>();
        int remainder = source.size() % n;  //(先计算出余数)
        int number = source.size() / n;  //然后是商
        int offset = 0;//偏移量
        for (int i = 0; i < n; i++) {
            List<T> value;
            if (remainder > 0) {
                value = source.subList(i * number + offset, (i + 1) * number + offset + 1);
                remainder--;
                offset++;
            } else {
                value = source.subList(i * number + offset, (i + 1) * number + offset);
            }
            result.add(value);
        }
        return result;
    }


}

package com.zm;


import com.zm.entity.UserTestEntity;
import com.zm.forkjoin.BatchTask;
import com.zm.mapper.UserTestMapper;
import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;

import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.ForkJoinTask;

@Slf4j
@SpringBootTest
class SpringMybatisApplicationTests {

    @Autowired
    private UserTestMapper userTestMapper;

    @Test
    void contextLoads() {
        List<UserTestEntity> userTestEntities = test1();
        log.info("=======开始========");
        ForkJoinPool pool = null;
        try {
            pool = new ForkJoinPool(10);
            BatchTask task = new BatchTask(userTestEntities, userTestMapper);
            ForkJoinTask<Void> submit = pool.submit(task);
            submit.get();
        } catch (InterruptedException e) {
            e.printStackTrace();
        } catch (ExecutionException e) {
            e.printStackTrace();
        } finally {
            // 关闭线程池
            pool.shutdown();
        }
        log.info("=======结束========");
    }

    /**
     * 初始化数据
     */
    private List<UserTestEntity> test1() {
        List<UserTestEntity> userTestEntities = new ArrayList<>();
        for (int i = 0; i < 1000000; i++) {
            UserTestEntity userTestEntity = new UserTestEntity();
            userTestEntity.setName1("2332131233");
            userTestEntity.setName2("3234343434");
            userTestEntity.setName3("3234343434");
            userTestEntity.setName4("3234343434");
            userTestEntity.setName5("3234343434");
            userTestEntity.setName6("3234343434");
            userTestEntity.setName7("3234343434");
            userTestEntity.setName8("3234343434");
            userTestEntity.setName9("3234343434");
            userTestEntities.add(userTestEntity);
        }
        return userTestEntities;
    }


}

100万数据插入花费时间大概8分钟左右~

本地电脑配置 ，mysql 5.7版本