#数据迁移#
一、创建多数据源项目
二、创建迁移Controller
其中业务逻辑 数量 自行调整
package com.example.demo.controller;
import com.example.demo.pojo.entity.LiveSettleEntity;
import com.example.demo.pojo.entity.ServicerInfoEntity;
import com.example.demo.service.SchoolServiceImpl;
import com.example.demo.service.TradeServiceImpl;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
/**
* 多线程同步历史数据
* @author zsf
* @date 2023-11-24
*/
@Slf4j
@RestController
@RequestMapping("/demo")
public class SynchronizeDataController {
@Autowired
private SchoolServiceImpl schoolService;
@Autowired
private TradeServiceImpl tradeService;
int batchSize = 10; // 每个批次的数据量
long totalCount = 100; // 要迁移的总数据量
int threadCount = 4; // 线程数
/**
* 多线程同步数据
* @param params
* @return
* @throws Exception
*/
@GetMapping("/syncData")
public void syncData(Map<String, Object> params) throws Exception {
ExecutorService executor = Executors.newFixedThreadPool(threadCount);
try {
// 设置源数据库查询的起始位置和每个批次的数据量
int offset = 0;
//获取表总数
totalCount=schoolService.count();
while (offset < totalCount) {
for (int i = 0; i < threadCount; i++) {
// 创建一个线程来处理每个批次的数据
executor.submit(new DataMigrationThread(offset, batchSize));
offset += batchSize;
}
}
} catch (Exception e) {
e.printStackTrace();
}
executor.shutdown();
try {
executor.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
private class DataMigrationThread implements Runnable {
private final int offset;//查询起始行数
private final int batchSize;//查询多少行
public DataMigrationThread(int offset, int batchSize) {
this.offset = offset;
this.batchSize = batchSize;
}
@Override
public void run() {
try {
// 查询数据并将其添加到批处理中
List<ServicerInfoEntity> queryList=schoolService.getList(offset,batchSize);
//写入数据
List<LiveSettleEntity> addList=new ArrayList<>();
//上限1000条
tradeService.saveBatch(addList);
} catch (Exception e) {
e.printStackTrace();
}
}
}
}
ServiceImpl
package com.example.demo.service;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.example.demo.mapper1.SchoolMapper;
import com.example.demo.pojo.entity.ServicerInfoEntity;
import org.springframework.stereotype.Service;
import java.util.List;
@Service
public class SchoolServiceImpl extends ServiceImpl<SchoolMapper, ServicerInfoEntity> {
public List<ServicerInfoEntity> getList(int offset,int batchSize){
return baseMapper.getList(offset,batchSize);
}
}
Mapper
package com.example.demo.mapper1;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.example.demo.pojo.entity.ServicerInfoEntity;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
import java.util.List;
@Mapper
public interface SchoolMapper extends BaseMapper<ServicerInfoEntity> {
List<ServicerInfoEntity> getList(@Param("offset") int offset, @Param("batchSize") int batchSize);
}
xml (路径自行修改)
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE mapper
PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
"https://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.example.demo.mapper1.SchoolMapper">
<resultMap id="resultMap" type="com.example.demo.pojo.entity.xxxxx">
</resultMap>
<select id="getList" resultType="com.example.demo.pojo.entity.xxxxx">
select * from xxxxx order by id desc
limit #{offset},#{batchSize}
</select>
</mapper>