定时任务+设计模式+线程池实现数据同步

概述:

        本文章目的在于实现数据同步,这个数据指的是不同系统,不同数据库之间的数据同步。

流程:

        先用文字描述,后面用图片补充

两个系统一个是商品系统,一个是订单系统,在订单系统里面,调用定时任务,拉取商品系统里的商品数据。

代码:

        订单系统定时任务

@Component
public class SyncDataQuartz {

    @Resource
    private TOmsSyncConfigService tOmsSyncConfigService;

    /**
     * @Description 同步PRD数据(全量)
     **/
    public void syncPrdDataFull(){
        // syncType '同步方式 0:全量,1:增量'
        tOmsSyncConfigService.syncPrdDataBySyncType(0);
    }

    /**
     * @Description 同步PRD数据(增量)
     **/
    public void syncPrdDataIncr(){
        // syncType '同步方式 0:全量,1:增量'
        tOmsSyncConfigService.syncPrdDataBySyncType(1);
    }

}

        数据同步配置表service

public interface TOmsSyncConfigService extends IService<TOmsSyncConfigEntity> {
    
    /**
     * @Description 根据同步方式同步Prd数据
     * @Param [syncType]
     **/
    void syncPrdDataBySyncType(int syncType);

    /**
     * @Description 根据Code同步数据
     * @Param [code]
     **/
    void syncDataByCode(String code);

    /**
     * @Description 请求PRD同步
     * @Param [tOmsSyncConfigEntity]
     **/
    ResponseEntity<String> syncPrdRequest(TOmsSyncConfigEntity tOmsSyncConfigEntity);

    /**
     * @Description 调用存储过程
     * @Param [procName,procParams]
     **/
    void callProc(String procName, LinkedHashMap procParams);

    /**
     * @Description 从PRD同步数据
     * @Param [tOmsSyncConfigEntity, syncDataPrd2OmsTemplate]
     **/
    boolean syncDataByPrd(TOmsSyncConfigEntity tOmsSyncConfigEntity, SyncDataPrd2OmsTemplate syncDataPrd2OmsTemplate);

    void callMergeProc(List<Integer> list) ;

    List<TCrmCustomerSyncTempEntity> selectCustomerFromCrm();
}

        同步配置表

public class TOmsSyncConfigEntity implements Serializable {
    private static final long serialVersionUID = 1L;

    @TableId(type = IdType.AUTO)
    @ApiModelProperty(value = "主键")
    private Long id;

    @ApiModelProperty(value = "唯一标识")
    private String code;

    @ApiModelProperty(value = "同步方式 0:全量,1:增量")
    private Integer syncType;

    @ApiModelProperty(value = "最新一次更新时间")
    private String latestSyncTime;

    @ApiModelProperty(value = "同步identity")
    private String syncIdentity;

    @ApiModelProperty(value = "是否开启")
    private Integer isEnable;

    @ApiModelProperty(value = "描述")
    private String description;

    @ApiModelProperty(value = "存储过程名称")
    private String procName;

    @ApiModelProperty(value = "请求url")
    private String requestUrl;

    @ApiModelProperty(value = "创建人id 创建人唯一标识")
    private String createBy;

    @ApiModelProperty(value = "创建时间 创建时间")
    private Date createTime;

    @ApiModelProperty(value = "修改人id 修改人唯一标识")
    private String updateBy;

    @ApiModelProperty(value = "修改时间 修改时间")
    private Date updateTime;

    @ApiModelProperty(value = "状态 默认:0  0=有效 1=无效")
    private Integer status;

    @ApiModelProperty(value = "同步页码")
    @TableField(exist = false)
    private int syncPageNo;

    @ApiModelProperty(value = "同步页大小")
    @TableField(exist = false)
    private int syncPageSize;

    @ApiModelProperty(value = "最大流水号(工厂同步使用)")
    @TableField(exist = false)
    private int maxSerialNumber;

    public String printSyncInfo() {
        return "同步页码:" + this.syncPageNo + ";同步页大小:" + this.syncPageSize +
                ";同步Code:" + this.code + ";同步存储过程名称:" + this.procName +
                ";最新一次更新时间:" + this.latestSyncTime + ";描述:" + this.description;
    }
}

        实现类

public class TOmsSyncConfigServiceImpl extends ServiceImpl<TOmsSyncConfigMapper, TOmsSyncConfigEntity> implements TOmsSyncConfigService {

    @Resource
    private TOmsSyncConfigService tOmsSyncConfigService;

    @Resource
    private TOmsSyncLogService tOmsSyncLogService;

    @Override
    public void syncPrdDataBySyncType(int syncType) {

        // 拿到所有启用的数据同步配置信息
        LambdaQueryWrapper<TOmsSyncConfigEntity> qw1 = new LambdaQueryWrapper<>();
        qw1.eq(TOmsSyncConfigEntity::getIsEnable, true);
        qw1.eq(TOmsSyncConfigEntity::getSyncType, syncType);
        qw1.eq(TOmsSyncConfigEntity::getStatus, 0);
        List<TOmsSyncConfigEntity> syncConfigEntities = this.baseMapper.selectList(qw1);

        // 使用策略工厂 + 线程池调用同步方法
        syncConfigEntities.stream().forEach(syncConfigEntity -> {
            SyncDataHandler syncDataHandler = SyncDataHandlerFactory.getInstance().getHandlerByType(syncConfigEntity.getCode());
            if (syncDataHandler != null) {
                SyncDataThreadPoolUtil.addTask(new SyncDataThreadPoolUtil.SyncDataTask(syncConfigEntity.getDescription()) {
                    @Override
                    public void run() {
                        syncDataHandler.syncData(syncConfigEntity);
                    }
                });
            }
        });

    }

@Override
    public boolean syncDataByPrd(TOmsSyncConfigEntity tOmsSyncConfigEntity, SyncDataPrd2OmsTemplate syncDataPrd2OmsTemplate) {
        // 声明批次ID
        String batchId = UuidUtils.getUUID();
        // 记录同步日志
        TOmsSyncLogEntity syncLogEntity = new TOmsSyncLogEntity();
        syncLogEntity.setSyncIdentity(tOmsSyncConfigEntity.getSyncIdentity());
        syncLogEntity.setSyncTime(new Date());
        syncLogEntity.setSyncBatchId(batchId);
        tOmsSyncLogService.save(syncLogEntity);

        try {
            // 以2000为批次做数据同步
            tOmsSyncConfigEntity.setSyncPageNo(0);
            tOmsSyncConfigEntity.setSyncPageSize(2000);
            int syncCount = 0; // 同步总数量
            int resultCount = 0; // 单次同步数量

            String currentDate = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(DateUtils.getDate());
            do {
                ResponseEntity<String> responseEntity = null;

                long timeMillisStart = System.currentTimeMillis();
                // 如果调用失败,重试请求,最多5次
                int retryCount = 5;
                while (retryCount != 0) {
                    tOmsSyncConfigEntity.setSyncPageNo(tOmsSyncConfigEntity.getSyncPageNo() + 1);
                    try {
                        // 调用同步接口
                        responseEntity = tOmsSyncConfigService.syncPrdRequest(tOmsSyncConfigEntity);
                        log.info("responseEntity:{}", responseEntity.getBody());
                    } catch (Exception e) {
                        log.error(tOmsSyncConfigEntity.getDescription() + "同步失败,第" + (5 - retryCount + 1) + "次请求,错误信息:" + e.toString());
                        // 休眠五秒,再次请求
                        Thread.sleep(1000 * 5);
                        retryCount--;
                        continue;
                    }
                    retryCount = 0;
                }
                long timeMillisEnd = System.currentTimeMillis();
                log.info("请求PRD耗时:{}", timeMillisEnd - timeMillisStart);

                if (responseEntity == null) {
                    throw new JeecgBootException("同步失败");
                }

                // BatchSave临时表
                ObjectMapper mapper = new ObjectMapper();
                Result result = mapper.readValue(responseEntity.getBody(), Result.class);

                List<LinkedHashMap<String, Object>> resultList = (List<LinkedHashMap<String, Object>>) result.getResult();

                if (resultList != null) {
                    // 模板方法模式调用同步Handler
                    syncDataPrd2OmsTemplate.saveBatchSyncTemp(resultList, batchId);
                }

                resultCount = resultList != null ? resultList.size() : 0;

                syncCount += resultCount;
            } while (resultCount == tOmsSyncConfigEntity.getSyncPageSize()); // 如拿满批次数据,再次请求


            // 更新同步总量
            syncLogEntity.setSyncCount(syncCount);
            tOmsSyncLogService.updateById(syncLogEntity);

            // 声明调用参数
            LinkedHashMap<String, Object> procParams = new LinkedHashMap();
            procParams.put("syncBatchId", batchId);
            procParams.put("syncLogId", syncLogEntity.getId());
            // 调用Proc增量更新OMS数据
            tOmsSyncConfigService.callProc(tOmsSyncConfigEntity.getProcName(), procParams);

            // 获取日志实体
            syncLogEntity = tOmsSyncLogService.getById(syncLogEntity.getId());

            // 回写上一次更新时间
            tOmsSyncConfigEntity.setLatestSyncTime(currentDate);
            tOmsSyncConfigService.updateById(tOmsSyncConfigEntity);
            // 回写同步状态
            syncLogEntity.setSyncResult(1);
            syncLogEntity.setSyncDetails("同步成功");
            tOmsSyncLogService.updateById(syncLogEntity);
        } catch (Exception e) {
            syncLogEntity.setSyncResult(0);
            syncLogEntity.setSyncDetails(e.getMessage());
            tOmsSyncLogService.updateById(syncLogEntity);
            log.error(e.toString());
            return false;
        }
        return true;
    }

}

        数据同步工厂

public class SyncDataHandlerFactory {
    private static Map<String, SyncDataHandler> syncDataMapHandlerMap = Maps.newConcurrentMap();

    private static class SyncDataFactoryHolder {
        private static final SyncDataHandlerFactory INSTANCE = new SyncDataHandlerFactory();
    }

    private SyncDataHandlerFactory(){}

    public static SyncDataHandlerFactory getInstance() {
        return SyncDataFactoryHolder.INSTANCE;
    }

    /**
     * @Description 根据类型获取handler
     * @Date 9:35 2023/4/25
     * @Param [type]
     * @return org.jeecg.modules.omsnew.sync.handler.SyncDataHandler
     **/
    public SyncDataHandler getHandlerByType(String type) {
        return syncDataMapHandlerMap.get(type);
    }

    /**
     * @return void
     * @Description 注册handler处理类
     * @Date 9:35 2023/4/25
     * @Param [type, syncDataHandler]
     **/
    public void register(String type, SyncDataHandler syncDataHandler) {
        //参数判空
        if (type == null || syncDataHandler == null) {
            return;
        }
        syncDataMapHandlerMap.put(type, syncDataHandler);
    }


}

        数据同步公共接口

public interface SyncDataHandler {

    /**
     * @return java.lang.Boolean
     * @Description 同步数据公用接口
     * @Date 10:42 2023/4/25
     * @Param [tOmsSyncConfigEntity]
     **/
    Boolean syncData(TOmsSyncConfigEntity tOmsSyncConfigEntity);

}

        数据同步线程池

public class SyncDataThreadPoolUtil {

    static ThreadPoolExecutor poolExecutor;
    static int addErrorCount = 0;

    static class MyRejectPolicy implements RejectedExecutionHandler {
        @Override
        public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {
            if (r instanceof SyncDataThread) {
                addErrorCount++;
                SyncDataThread syncDataThread = (SyncDataThread) r;
                log.info(addErrorCount + ":拒绝处理任务: 内容:" + syncDataThread.task.description);
            }
        }
    }

    static {
        ThreadFactory factory = new CustomizableThreadFactory("sync-data-pool-");
        poolExecutor = new ThreadPoolExecutor(40,
                40,
                60L,
                TimeUnit.SECONDS,
                new ArrayBlockingQueue<>(500), factory, new MyRejectPolicy());
    }

    static class SyncDataThread<T> extends Thread {
        SyncDataTask task;

        public SyncDataThread(SyncDataTask task) {
            this.task = task;
        }

        @Override
        public void run() {
            String threadName = Thread.currentThread().getName();
            long threadId = Thread.currentThread().getId();
            int size = poolExecutor.getQueue().size();
            int activeCount = poolExecutor.getActiveCount();
            System.out.printf("线程名:%s 线程id:%s 剩余:%s 活跃线程:%s\r\n", threadName, threadId, size, activeCount);
            task.run();
        }
    }

    public static abstract class SyncDataTask {
        String description;

        public abstract void run();

        public SyncDataTask(String description) {
            this.description = description;
        }
    }

    public static <T> void addTask(SyncDataTask task) {
        poolExecutor.execute(new SyncDataThread<T>(task));
    }

}

        具体的产品表的策略

public class SyncPrdProductHandler extends SyncDataPrd2OmsTemplate implements SyncDataHandler, InitializingBean {

    @Resource
    private TOmsSyncConfigService tOmsSyncConfigService;

    @Resource
    private TPrdProductSyncTempService tPrdProductSyncTempService;

    @Override
    public Boolean syncData(TOmsSyncConfigEntity tOmsSyncConfigEntity) {
        return tOmsSyncConfigService.syncDataByPrd(tOmsSyncConfigEntity, this);
    }

    @Override
    public void afterPropertiesSet() {
        SyncDataHandlerFactory.getInstance().register(SyncDataTypeEnum.PRD_PRODUCT.getCode(), this);
    }

    @Override
    public void saveBatchSyncTemp(List<LinkedHashMap<String, Object>> resultList, String batchId) {
        long timeMillisStart = System.currentTimeMillis();
        // 将批次ID插入每条记录
        List<TPrdProductSyncTempEntity> batchSaveList = new ArrayList<>();
        for (LinkedHashMap<String, Object> temp : resultList) {
            TPrdProductSyncTempEntity tPrdProductSyncTempEntity = JSON.parseObject(JSON.toJSONString(temp), TPrdProductSyncTempEntity.class);
            tPrdProductSyncTempEntity.setSyncBatchId(batchId);
            batchSaveList.add(tPrdProductSyncTempEntity);
        }
        long timeMillisEnd = System.currentTimeMillis();
        tPrdProductSyncTempService.saveBatch(batchSaveList);
        log.info("产品数据 batchSave耗时:{}", timeMillisEnd - timeMillisStart);
    }
}

          同步表的模版方法

public abstract class SyncDataPrd2OmsTemplate {

    /**
     * @Description 批量保存同步临时数据
     * @Date 9:39 2023/4/28
     * @Param [resultList, batchId]
     * @return void
     **/
    public abstract void saveBatchSyncTemp(List<LinkedHashMap<String, Object>> resultList, String batchId);

}

        同步枚举类

package org.jeecg.modules.omsnew.sync.enumutils;

/**
 * @ClassName SyncDataTypeEnum
 * @Description 同步数据类型枚举
 * @Author LiQin.Huang
 * @Date 2023/4/25 9:42
 * @Version 1.0
 */
public enum SyncDataTypeEnum {

    PRD_ATTRIBUTE_PRODUCT_ELEMENT("syncPrdAttributeProductElement", "", "同步PRD元素大类"),
    PRD_PURITY("syncPrdPurity", "", "OMS同步PRD成色"),
    PRD_CHANNEL_COMMODITY("syncPrdChannelCommodity", "", "OMS同步PRD渠道商品"),
    PRD_FITTINGS("syncPrdFittings", "", "OMS同步PRD配件"),
    PRD_COUNTER("syncPrdCounter", "", "OMS同步PRD柜台"),
    PRD_CONFIG_SHADING("syncPrdConfigShading", "", "OMS同步PRD底纹"),
    PRD_CONFIG_LATHES_CARVED_FLOWERS("syncPrdConfigLathesCarvedFlowers", "", "OMS同步PRD车花款"),
    PRD_CONFIG_FINISHING_CRAFTSMANSHIP("syncPrdConfigFinishingCraftsmanship", "", "OMS同步PRD饰面工艺"),
    PRD_CONFIG_BRAND_NETWORK("syncPrdConfigBrandNetwork", "", "OMS同步PRD品牌展厅关系"),
    PRD_CONFIG_BRAND("syncPrdConfigBrand", "", "OMS同步PRD品牌"),
    PRD_CONFIG_ATTRIBUTE_CATEGORY("syncPrdConfigAttributeCategory", "", "OMS同步PRD五级分类"),
    PRD_CONFIG_ADDITIONAL_COST("syncPrdConfigAdditionalCost", "", "OMS同步PRD附加工费"),
    PRD_PRODUCT("syncPrdProduct", "", "OMS同步PRD产品数据"),
    PRD_COMMODITY("syncPrdCommodity", "", "OMS同步PRD商品数据"),
    PRD_SPECIFICATION_COMMODITY("syncPrdSpecificationCommodity", "", "OMS同步PRD规格关联数据"),
    PRD_COMMODITY_FITTINGS("syncPrdCommodityFittings", "", "OMS同步PRD配件关联数据"),
    PRD_SUPPLIER_COMMODITY("syncPrdSupplierCommodity", "", "OMS同步PRD供应商关联数据"),
    PRD_SPECIFICATION_ATTRIBUTE("syncPrdSpecificationAttribute", "", "OMS同步PRD商品规格数据"),
    PRD_SUPPLIER_PRODUCTION_NETWORK("syncPrdSupplierProductionNetwork", "", "OMS同步PRD供应商商数据"),
    PRD_COMMODITY_SALE_NETWORK("syncPrdCommoditySaleNetwork", "", "OMS同步PRD商品销售渠道数据"),
    PRD_CONFIG_SALE_NETWORK("syncPrdConfigSaleNetwork", "", "OMS同步PRD销售渠道数据"),
    PRD_CONFIG_PRODUCTION_NETWORK("syncPrdConfigProductionNetwork", "", "OMS同步PRD生产工厂"),
    PRD_SUIT_COMMODITY("syncPrdSuitCommodity", "", "OMS同步PRD套装商品中间表"),
    PRD_FACTORY_INFORMATION("syncPrdFactoryInformation", "", "OMS同步PRD工厂信息维护临时表"),
    FACTORY_CPCCDH("syncFactorySjdcdh", "", "OMS同步工厂素金调出单数据"),
    FACTORY_CPCCDB("syncFactorySjdcdb", "", "OMS同步工厂素金调出单明细数据"),

    CRM_CUSTOMER("syncCrmCustomer", "", "OMS同步CRM客户");





    private String code;

    private String identity;

    private String description;

    SyncDataTypeEnum(String code, String identity, String description) {
        this.code = code;
        this.identity = identity;
        this.description = description;
    }

    public String getCode() {
        return code;
    }

    public String getIdentity() {
        return identity;
    }

    public String getDescription() {
        return description;
    }
}

  • 5
    点赞
  • 3
    收藏
    觉得还不错? 一键收藏
  • 0
    评论
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值