多线程多数据源数据迁移

#数据迁移#

一、创建多数据源项目

查看多数据源demo创建说明

二、创建迁移Controller

其中业务逻辑 数量 自行调整

package com.example.demo.controller;


import com.example.demo.pojo.entity.LiveSettleEntity;
import com.example.demo.pojo.entity.ServicerInfoEntity;
import com.example.demo.service.SchoolServiceImpl;
import com.example.demo.service.TradeServiceImpl;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;

import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;

/**
 * 多线程同步历史数据
 * @author zsf
 * @date 2023-11-24
 */
@Slf4j
@RestController
@RequestMapping("/demo")
public class SynchronizeDataController {

    @Autowired
    private SchoolServiceImpl schoolService;
    @Autowired
    private TradeServiceImpl tradeService;

    int batchSize = 10; // 每个批次的数据量

    long totalCount = 100; // 要迁移的总数据量

    int threadCount = 4; // 线程数


    /**
     * 多线程同步数据
     * @param params
     * @return
     * @throws Exception
     */
    @GetMapping("/syncData")
    public void syncData(Map<String, Object> params) throws Exception {
        ExecutorService executor = Executors.newFixedThreadPool(threadCount);
        try {
            // 设置源数据库查询的起始位置和每个批次的数据量
            int offset = 0;
            //获取表总数
            totalCount=schoolService.count();
            while (offset < totalCount) {
                for (int i = 0; i < threadCount; i++) {
                    // 创建一个线程来处理每个批次的数据
                    executor.submit(new DataMigrationThread(offset, batchSize));
                    offset += batchSize;
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
        executor.shutdown();
        try {
            executor.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }

    private class DataMigrationThread implements Runnable {
        private final int offset;//查询起始行数
        private final int batchSize;//查询多少行
        public DataMigrationThread(int offset, int batchSize) {
            this.offset = offset;
            this.batchSize = batchSize;
        }

        @Override
        public void run() {
            try {
                // 查询数据并将其添加到批处理中
                List<ServicerInfoEntity> queryList=schoolService.getList(offset,batchSize);

                //写入数据
                List<LiveSettleEntity> addList=new ArrayList<>();
                //上限1000条
                tradeService.saveBatch(addList);

            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }
}

ServiceImpl

package com.example.demo.service;


import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.example.demo.mapper1.SchoolMapper;
import com.example.demo.pojo.entity.ServicerInfoEntity;
import org.springframework.stereotype.Service;

import java.util.List;

@Service
public class SchoolServiceImpl extends ServiceImpl<SchoolMapper, ServicerInfoEntity> {

    public List<ServicerInfoEntity> getList(int offset,int batchSize){
        return baseMapper.getList(offset,batchSize);
    }
}

Mapper

package com.example.demo.mapper1;

import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.example.demo.pojo.entity.ServicerInfoEntity;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;

import java.util.List;

@Mapper
public interface SchoolMapper extends BaseMapper<ServicerInfoEntity> {
    List<ServicerInfoEntity> getList(@Param("offset") int offset, @Param("batchSize") int batchSize);
}

xml (路径自行修改)

<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE mapper
        PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
        "https://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.example.demo.mapper1.SchoolMapper">
    <resultMap id="resultMap" type="com.example.demo.pojo.entity.xxxxx">
    </resultMap>

    <select id="getList" resultType="com.example.demo.pojo.entity.xxxxx">
        select * from xxxxx  order by id desc
        limit #{offset},#{batchSize}
    </select>

</mapper>