概述:
本文章目的在于实现数据同步,这个数据指的是不同系统,不同数据库之间的数据同步。
流程:
先用文字描述,后面用图片补充
两个系统一个是商品系统,一个是订单系统,在订单系统里面,调用定时任务,拉取商品系统里的商品数据。
代码:
订单系统定时任务
@Component
public class SyncDataQuartz {
@Resource
private TOmsSyncConfigService tOmsSyncConfigService;
/**
* @Description 同步PRD数据(全量)
**/
public void syncPrdDataFull(){
// syncType '同步方式 0:全量,1:增量'
tOmsSyncConfigService.syncPrdDataBySyncType(0);
}
/**
* @Description 同步PRD数据(增量)
**/
public void syncPrdDataIncr(){
// syncType '同步方式 0:全量,1:增量'
tOmsSyncConfigService.syncPrdDataBySyncType(1);
}
}
数据同步配置表service
public interface TOmsSyncConfigService extends IService<TOmsSyncConfigEntity> {
/**
* @Description 根据同步方式同步Prd数据
* @Param [syncType]
**/
void syncPrdDataBySyncType(int syncType);
/**
* @Description 根据Code同步数据
* @Param [code]
**/
void syncDataByCode(String code);
/**
* @Description 请求PRD同步
* @Param [tOmsSyncConfigEntity]
**/
ResponseEntity<String> syncPrdRequest(TOmsSyncConfigEntity tOmsSyncConfigEntity);
/**
* @Description 调用存储过程
* @Param [procName,procParams]
**/
void callProc(String procName, LinkedHashMap procParams);
/**
* @Description 从PRD同步数据
* @Param [tOmsSyncConfigEntity, syncDataPrd2OmsTemplate]
**/
boolean syncDataByPrd(TOmsSyncConfigEntity tOmsSyncConfigEntity, SyncDataPrd2OmsTemplate syncDataPrd2OmsTemplate);
void callMergeProc(List<Integer> list) ;
List<TCrmCustomerSyncTempEntity> selectCustomerFromCrm();
}
同步配置表
public class TOmsSyncConfigEntity implements Serializable {
private static final long serialVersionUID = 1L;
@TableId(type = IdType.AUTO)
@ApiModelProperty(value = "主键")
private Long id;
@ApiModelProperty(value = "唯一标识")
private String code;
@ApiModelProperty(value = "同步方式 0:全量,1:增量")
private Integer syncType;
@ApiModelProperty(value = "最新一次更新时间")
private String latestSyncTime;
@ApiModelProperty(value = "同步identity")
private String syncIdentity;
@ApiModelProperty(value = "是否开启")
private Integer isEnable;
@ApiModelProperty(value = "描述")
private String description;
@ApiModelProperty(value = "存储过程名称")
private String procName;
@ApiModelProperty(value = "请求url")
private String requestUrl;
@ApiModelProperty(value = "创建人id 创建人唯一标识")
private String createBy;
@ApiModelProperty(value = "创建时间 创建时间")
private Date createTime;
@ApiModelProperty(value = "修改人id 修改人唯一标识")
private String updateBy;
@ApiModelProperty(value = "修改时间 修改时间")
private Date updateTime;
@ApiModelProperty(value = "状态 默认:0 0=有效 1=无效")
private Integer status;
@ApiModelProperty(value = "同步页码")
@TableField(exist = false)
private int syncPageNo;
@ApiModelProperty(value = "同步页大小")
@TableField(exist = false)
private int syncPageSize;
@ApiModelProperty(value = "最大流水号(工厂同步使用)")
@TableField(exist = false)
private int maxSerialNumber;
public String printSyncInfo() {
return "同步页码:" + this.syncPageNo + ";同步页大小:" + this.syncPageSize +
";同步Code:" + this.code + ";同步存储过程名称:" + this.procName +
";最新一次更新时间:" + this.latestSyncTime + ";描述:" + this.description;
}
}
实现类
public class TOmsSyncConfigServiceImpl extends ServiceImpl<TOmsSyncConfigMapper, TOmsSyncConfigEntity> implements TOmsSyncConfigService {
@Resource
private TOmsSyncConfigService tOmsSyncConfigService;
@Resource
private TOmsSyncLogService tOmsSyncLogService;
@Override
public void syncPrdDataBySyncType(int syncType) {
// 拿到所有启用的数据同步配置信息
LambdaQueryWrapper<TOmsSyncConfigEntity> qw1 = new LambdaQueryWrapper<>();
qw1.eq(TOmsSyncConfigEntity::getIsEnable, true);
qw1.eq(TOmsSyncConfigEntity::getSyncType, syncType);
qw1.eq(TOmsSyncConfigEntity::getStatus, 0);
List<TOmsSyncConfigEntity> syncConfigEntities = this.baseMapper.selectList(qw1);
// 使用策略工厂 + 线程池调用同步方法
syncConfigEntities.stream().forEach(syncConfigEntity -> {
SyncDataHandler syncDataHandler = SyncDataHandlerFactory.getInstance().getHandlerByType(syncConfigEntity.getCode());
if (syncDataHandler != null) {
SyncDataThreadPoolUtil.addTask(new SyncDataThreadPoolUtil.SyncDataTask(syncConfigEntity.getDescription()) {
@Override
public void run() {
syncDataHandler.syncData(syncConfigEntity);
}
});
}
});
}
@Override
public boolean syncDataByPrd(TOmsSyncConfigEntity tOmsSyncConfigEntity, SyncDataPrd2OmsTemplate syncDataPrd2OmsTemplate) {
// 声明批次ID
String batchId = UuidUtils.getUUID();
// 记录同步日志
TOmsSyncLogEntity syncLogEntity = new TOmsSyncLogEntity();
syncLogEntity.setSyncIdentity(tOmsSyncConfigEntity.getSyncIdentity());
syncLogEntity.setSyncTime(new Date());
syncLogEntity.setSyncBatchId(batchId);
tOmsSyncLogService.save(syncLogEntity);
try {
// 以2000为批次做数据同步
tOmsSyncConfigEntity.setSyncPageNo(0);
tOmsSyncConfigEntity.setSyncPageSize(2000);
int syncCount = 0; // 同步总数量
int resultCount = 0; // 单次同步数量
String currentDate = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(DateUtils.getDate());
do {
ResponseEntity<String> responseEntity = null;
long timeMillisStart = System.currentTimeMillis();
// 如果调用失败,重试请求,最多5次
int retryCount = 5;
while (retryCount != 0) {
tOmsSyncConfigEntity.setSyncPageNo(tOmsSyncConfigEntity.getSyncPageNo() + 1);
try {
// 调用同步接口
responseEntity = tOmsSyncConfigService.syncPrdRequest(tOmsSyncConfigEntity);
log.info("responseEntity:{}", responseEntity.getBody());
} catch (Exception e) {
log.error(tOmsSyncConfigEntity.getDescription() + "同步失败,第" + (5 - retryCount + 1) + "次请求,错误信息:" + e.toString());
// 休眠五秒,再次请求
Thread.sleep(1000 * 5);
retryCount--;
continue;
}
retryCount = 0;
}
long timeMillisEnd = System.currentTimeMillis();
log.info("请求PRD耗时:{}", timeMillisEnd - timeMillisStart);
if (responseEntity == null) {
throw new JeecgBootException("同步失败");
}
// BatchSave临时表
ObjectMapper mapper = new ObjectMapper();
Result result = mapper.readValue(responseEntity.getBody(), Result.class);
List<LinkedHashMap<String, Object>> resultList = (List<LinkedHashMap<String, Object>>) result.getResult();
if (resultList != null) {
// 模板方法模式调用同步Handler
syncDataPrd2OmsTemplate.saveBatchSyncTemp(resultList, batchId);
}
resultCount = resultList != null ? resultList.size() : 0;
syncCount += resultCount;
} while (resultCount == tOmsSyncConfigEntity.getSyncPageSize()); // 如拿满批次数据,再次请求
// 更新同步总量
syncLogEntity.setSyncCount(syncCount);
tOmsSyncLogService.updateById(syncLogEntity);
// 声明调用参数
LinkedHashMap<String, Object> procParams = new LinkedHashMap();
procParams.put("syncBatchId", batchId);
procParams.put("syncLogId", syncLogEntity.getId());
// 调用Proc增量更新OMS数据
tOmsSyncConfigService.callProc(tOmsSyncConfigEntity.getProcName(), procParams);
// 获取日志实体
syncLogEntity = tOmsSyncLogService.getById(syncLogEntity.getId());
// 回写上一次更新时间
tOmsSyncConfigEntity.setLatestSyncTime(currentDate);
tOmsSyncConfigService.updateById(tOmsSyncConfigEntity);
// 回写同步状态
syncLogEntity.setSyncResult(1);
syncLogEntity.setSyncDetails("同步成功");
tOmsSyncLogService.updateById(syncLogEntity);
} catch (Exception e) {
syncLogEntity.setSyncResult(0);
syncLogEntity.setSyncDetails(e.getMessage());
tOmsSyncLogService.updateById(syncLogEntity);
log.error(e.toString());
return false;
}
return true;
}
}
数据同步工厂
public class SyncDataHandlerFactory {
private static Map<String, SyncDataHandler> syncDataMapHandlerMap = Maps.newConcurrentMap();
private static class SyncDataFactoryHolder {
private static final SyncDataHandlerFactory INSTANCE = new SyncDataHandlerFactory();
}
private SyncDataHandlerFactory(){}
public static SyncDataHandlerFactory getInstance() {
return SyncDataFactoryHolder.INSTANCE;
}
/**
* @Description 根据类型获取handler
* @Date 9:35 2023/4/25
* @Param [type]
* @return org.jeecg.modules.omsnew.sync.handler.SyncDataHandler
**/
public SyncDataHandler getHandlerByType(String type) {
return syncDataMapHandlerMap.get(type);
}
/**
* @return void
* @Description 注册handler处理类
* @Date 9:35 2023/4/25
* @Param [type, syncDataHandler]
**/
public void register(String type, SyncDataHandler syncDataHandler) {
//参数判空
if (type == null || syncDataHandler == null) {
return;
}
syncDataMapHandlerMap.put(type, syncDataHandler);
}
}
数据同步公共接口
public interface SyncDataHandler {
/**
* @return java.lang.Boolean
* @Description 同步数据公用接口
* @Date 10:42 2023/4/25
* @Param [tOmsSyncConfigEntity]
**/
Boolean syncData(TOmsSyncConfigEntity tOmsSyncConfigEntity);
}
数据同步线程池
public class SyncDataThreadPoolUtil {
static ThreadPoolExecutor poolExecutor;
static int addErrorCount = 0;
static class MyRejectPolicy implements RejectedExecutionHandler {
@Override
public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {
if (r instanceof SyncDataThread) {
addErrorCount++;
SyncDataThread syncDataThread = (SyncDataThread) r;
log.info(addErrorCount + ":拒绝处理任务: 内容:" + syncDataThread.task.description);
}
}
}
static {
ThreadFactory factory = new CustomizableThreadFactory("sync-data-pool-");
poolExecutor = new ThreadPoolExecutor(40,
40,
60L,
TimeUnit.SECONDS,
new ArrayBlockingQueue<>(500), factory, new MyRejectPolicy());
}
static class SyncDataThread<T> extends Thread {
SyncDataTask task;
public SyncDataThread(SyncDataTask task) {
this.task = task;
}
@Override
public void run() {
String threadName = Thread.currentThread().getName();
long threadId = Thread.currentThread().getId();
int size = poolExecutor.getQueue().size();
int activeCount = poolExecutor.getActiveCount();
System.out.printf("线程名:%s 线程id:%s 剩余:%s 活跃线程:%s\r\n", threadName, threadId, size, activeCount);
task.run();
}
}
public static abstract class SyncDataTask {
String description;
public abstract void run();
public SyncDataTask(String description) {
this.description = description;
}
}
public static <T> void addTask(SyncDataTask task) {
poolExecutor.execute(new SyncDataThread<T>(task));
}
}
具体的产品表的策略
public class SyncPrdProductHandler extends SyncDataPrd2OmsTemplate implements SyncDataHandler, InitializingBean {
@Resource
private TOmsSyncConfigService tOmsSyncConfigService;
@Resource
private TPrdProductSyncTempService tPrdProductSyncTempService;
@Override
public Boolean syncData(TOmsSyncConfigEntity tOmsSyncConfigEntity) {
return tOmsSyncConfigService.syncDataByPrd(tOmsSyncConfigEntity, this);
}
@Override
public void afterPropertiesSet() {
SyncDataHandlerFactory.getInstance().register(SyncDataTypeEnum.PRD_PRODUCT.getCode(), this);
}
@Override
public void saveBatchSyncTemp(List<LinkedHashMap<String, Object>> resultList, String batchId) {
long timeMillisStart = System.currentTimeMillis();
// 将批次ID插入每条记录
List<TPrdProductSyncTempEntity> batchSaveList = new ArrayList<>();
for (LinkedHashMap<String, Object> temp : resultList) {
TPrdProductSyncTempEntity tPrdProductSyncTempEntity = JSON.parseObject(JSON.toJSONString(temp), TPrdProductSyncTempEntity.class);
tPrdProductSyncTempEntity.setSyncBatchId(batchId);
batchSaveList.add(tPrdProductSyncTempEntity);
}
long timeMillisEnd = System.currentTimeMillis();
tPrdProductSyncTempService.saveBatch(batchSaveList);
log.info("产品数据 batchSave耗时:{}", timeMillisEnd - timeMillisStart);
}
}
同步表的模版方法
public abstract class SyncDataPrd2OmsTemplate {
/**
* @Description 批量保存同步临时数据
* @Date 9:39 2023/4/28
* @Param [resultList, batchId]
* @return void
**/
public abstract void saveBatchSyncTemp(List<LinkedHashMap<String, Object>> resultList, String batchId);
}
同步枚举类
package org.jeecg.modules.omsnew.sync.enumutils;
/**
* @ClassName SyncDataTypeEnum
* @Description 同步数据类型枚举
* @Author LiQin.Huang
* @Date 2023/4/25 9:42
* @Version 1.0
*/
public enum SyncDataTypeEnum {
PRD_ATTRIBUTE_PRODUCT_ELEMENT("syncPrdAttributeProductElement", "", "同步PRD元素大类"),
PRD_PURITY("syncPrdPurity", "", "OMS同步PRD成色"),
PRD_CHANNEL_COMMODITY("syncPrdChannelCommodity", "", "OMS同步PRD渠道商品"),
PRD_FITTINGS("syncPrdFittings", "", "OMS同步PRD配件"),
PRD_COUNTER("syncPrdCounter", "", "OMS同步PRD柜台"),
PRD_CONFIG_SHADING("syncPrdConfigShading", "", "OMS同步PRD底纹"),
PRD_CONFIG_LATHES_CARVED_FLOWERS("syncPrdConfigLathesCarvedFlowers", "", "OMS同步PRD车花款"),
PRD_CONFIG_FINISHING_CRAFTSMANSHIP("syncPrdConfigFinishingCraftsmanship", "", "OMS同步PRD饰面工艺"),
PRD_CONFIG_BRAND_NETWORK("syncPrdConfigBrandNetwork", "", "OMS同步PRD品牌展厅关系"),
PRD_CONFIG_BRAND("syncPrdConfigBrand", "", "OMS同步PRD品牌"),
PRD_CONFIG_ATTRIBUTE_CATEGORY("syncPrdConfigAttributeCategory", "", "OMS同步PRD五级分类"),
PRD_CONFIG_ADDITIONAL_COST("syncPrdConfigAdditionalCost", "", "OMS同步PRD附加工费"),
PRD_PRODUCT("syncPrdProduct", "", "OMS同步PRD产品数据"),
PRD_COMMODITY("syncPrdCommodity", "", "OMS同步PRD商品数据"),
PRD_SPECIFICATION_COMMODITY("syncPrdSpecificationCommodity", "", "OMS同步PRD规格关联数据"),
PRD_COMMODITY_FITTINGS("syncPrdCommodityFittings", "", "OMS同步PRD配件关联数据"),
PRD_SUPPLIER_COMMODITY("syncPrdSupplierCommodity", "", "OMS同步PRD供应商关联数据"),
PRD_SPECIFICATION_ATTRIBUTE("syncPrdSpecificationAttribute", "", "OMS同步PRD商品规格数据"),
PRD_SUPPLIER_PRODUCTION_NETWORK("syncPrdSupplierProductionNetwork", "", "OMS同步PRD供应商商数据"),
PRD_COMMODITY_SALE_NETWORK("syncPrdCommoditySaleNetwork", "", "OMS同步PRD商品销售渠道数据"),
PRD_CONFIG_SALE_NETWORK("syncPrdConfigSaleNetwork", "", "OMS同步PRD销售渠道数据"),
PRD_CONFIG_PRODUCTION_NETWORK("syncPrdConfigProductionNetwork", "", "OMS同步PRD生产工厂"),
PRD_SUIT_COMMODITY("syncPrdSuitCommodity", "", "OMS同步PRD套装商品中间表"),
PRD_FACTORY_INFORMATION("syncPrdFactoryInformation", "", "OMS同步PRD工厂信息维护临时表"),
FACTORY_CPCCDH("syncFactorySjdcdh", "", "OMS同步工厂素金调出单数据"),
FACTORY_CPCCDB("syncFactorySjdcdb", "", "OMS同步工厂素金调出单明细数据"),
CRM_CUSTOMER("syncCrmCustomer", "", "OMS同步CRM客户");
private String code;
private String identity;
private String description;
SyncDataTypeEnum(String code, String identity, String description) {
this.code = code;
this.identity = identity;
this.description = description;
}
public String getCode() {
return code;
}
public String getIdentity() {
return identity;
}
public String getDescription() {
return description;
}
}