冗余数据同步方案

商品用户客户员工店铺仓库供应商这些数据修改后，需要同步到历史快照订单，发货单，订货单，采购单，调拨单，出库入库单等等....，之前的逻辑是直接在update接口里面加上synchronousUpdate方法，同步更新对应的表，有时候一个业务的update可能对应二十多张目标表，完全耦合在了业务里，接口执行的效率和代码的可维护性都不强

生产是在Handler里做的（THandler）

把这套逻辑通过canal监控的模式拆分出来，监控我们对应的原表，完全与业务进行解耦合通过@CanalTable注解监控对应的表，实现EntryHandler<T>,重写对应的update方法，这样原表一修改以后，canal就会来调我们的update方法，canal会在update方法里给我们 (T before，T after)的入参，这样我们就能拿到原表修改前后修改后的数据

通过一个通用的工具类EnableCompareUtils，通过getClass().getName()方法拿到className，去查询我们的redundant_source表，redundant_source表name记录了同步业务名称，sync_field记录了我们需要监控的字段，比如table_name:customer name:客户信息同步 sync_field:["name","phone"] 表示监控客户信息表customer的name，phone字段

从canal的容器CanalContext中拿到原表中实际更新过的字段updateField，用syncFields.contains(uf)比较updateField中哪些字段需要做冗余同步如果有的话取到SyncId向redundant_update表中插入一条数据同步状态为WAIT(1,"待同步")

自此生产过程结束

消费是在redundant-sync模块中做的

ScheduledTasks每100ms去redundant_update表中拉一条数据，把同步状态改为SYNCHRONIZING(2,"同步中")，之后通过

ApplicationEventPublisher去发布事件，监听者RedundantUpdateListener通过查询redundant_target表找出全部需要冗余同步的表，去做实际的更新逻辑，redundant_target表中的set_params/search_params是json格式，通过sqlAssemble去解析并交给dynamicService在Mapper层拼接sql动态更新具体的表，更新成功后把同步状态改为SUCCESS(3,"同步成功")，如果更新的过程报错了，用catch语句块捕获住异常，把同步状态改为FAIR(4,"同步失败")，最后把日志记录到redundant_sync_log表里，这张表记录了set_str,search_str,sync_date,source_id,target_id_update_id等信息，方便我们后期排查维护

自此消费过程结束

yeb-canal-support模块：生产者

引入canal监听对应表的binlog日志

创建TableNameHandler类实现canal-client 提供的 EntryHandler<T> 类来实现对于数据表的监控，从而达到数据的增删改同步

通过@CanalTable(value = "table_name")指定监控的表

@Component
@Slf4j
@CanalTable(value = "goods_info")
public class GoodsInfoHandler implements EntryHandler<GoodsInfo> {
    @Autowired
    private EnableCompareUtils enableCompareUtils;

    @Override
    public void insert(GoodsInfo goodsInfo) {
        log.info("insert message  {}", goodsInfo);
    }

    @Override
    public void update(GoodsInfo before, GoodsInfo after) {
        log.info("update before {} ", before);
        log.info("update after {}", after);
        enableCompareUtils.dataSync(before,after);
    }



    @Override
    public void delete(GoodsInfo goodsInfo) {
        log.info("delete  {}", goodsInfo);
    }


}

数据库主表更新后，canal会监听到并走update的逻辑

通过自定义的EnableCompareUtils工具类调用dataSyn方法

public void dataSync(Object before, Object after) {
  try {
    //拿到data数据
    List<Map<String, String>> data = CanalContext.getModel().getData();
    //有数据才记录
    if (CollectionUtil.isNotEmpty(data) && data.size() > 1) {
      //拿到原表修改的字段
      Set<String> updateField = data.get(1).keySet();
      //更新后的字段数据
      Map<String, String> afterDataMap = data.get(0);
      //拿到更新表的实体对象
      Class<?> afterClass = after.getClass();
      //通过class从数据库查询出冗余源对象
      RedundantSourceDTO redundantSourceDTO = redundantSourceReadFacade.getByClassName(afterClass.getName());
      //判断该class是否有冗余
      if (redundantSourceDTO != null && ObjectUtil.isNotEmpty(redundantSourceDTO.getSyncField())) {
        //有冗余
        //拿到需要更新的字段
        List<String> syncFields = JSON.parseArray(redundantSourceDTO.getSyncField(), String.class);
        for (String uf : updateField) {
          //判断原表修改的字段需要冗余的字段拿出来
          if (syncFields.contains(uf)) {
            //组装数据
            RedundantSyncAO redundantSyncAO = new RedundantSyncAO();
            redundantSyncAO.setSyncId(Long.valueOf(afterDataMap.get(ID)));
      redundantSyncAO.setTenantId(Long.valueOf(afterDataMap.get(TENANT_ID)));
            redundantSyncAO.setClassName(afterClass.getName());
            //增加一条记录到redundant_update表里，等待定时任务监控
            //同步状态为WAIT(1,"待同步")
            Boolean save = redundantUpdateWriteFacade.save(redundantSyncAO);
            log.info("EnableCompareUtils dataSync save状态={}", save);
            break;
          }
        }
      }
    }
  } catch (Exception e) {
    log.error("EnableCompareUtils dataSync error", e);
  }
}

从canal的容器CanalContext里拿到data数据

//拿到data数据
    List<Map<String, String>> data = CanalContext.getModel().getData();

/**
 * FlatMessage形式维护
 * before and after数据 index:0-after；1-before
 */
private List<Map<String, String>> data;

redundant_sync模块：消费者

@Component
public class ScheduledTasks {

    @Resource
    private SyncService syncService;

    @Resource
    private RedundantUpdateService redundantUpdateService;

    @Scheduled(fixedRate = 100)
    public void syncData() {
      //根据优先级排序，把redundant_update表里状态为WAIT的一条一条拿出来
      //状态改为SYNCHRONIZING(2,"同步中")
        RedundantSync syncDate = redundantUpdateService.getSyncData();
        if (syncDate != null) {
          //同步逻辑
            syncService.getById(syncDate);
        }
    }
}

ScheduledTasks起一个定时任务，100ms执行一次

    @Override
    public void getById(RedundantSync redundantSync) {
      //动态根据数据库名，表名，ID，查询最新的数据
        Map<String, Object> byIdDynamic = dynamicService.getByIdDynamic(redundantSync);
        if (CollectionUtil.isNotEmpty(byIdDynamic)) {
            redundantSync.setSourceMap(byIdDynamic);
        }
        //基于spring，发布事件
        this.publishSearchEvent(redundantSync);
    }

发布事件

@Component
@Slf4j
//@Async
public class RedundantUpdateListener implements ApplicationListener<RedundantUpdateEvent> {

    @Resource
    private SyncService syncService;

    @Override
    public void onApplicationEvent(@NonNull RedundantUpdateEvent event) {
        log.info("同步信息!  {}", JSONObject.toJSONString(event));
        log.info("发送消息内容：{}", JSONObject.toJSONString(event.getRedundantSync()));
        RedundantSync redundantSync = event.getRedundantSync();
        syncService.select(redundantSync);
    }
}

RedundantUpdateListener监听

@Override
public void select(RedundantSync redundantSync) {
    RedundantUpdateDO redundantUpdateDO = new RedundantUpdateDO();
    redundantUpdateDO.setId(redundantSync.getId());
    //通过RedundantSourceId拿到所有的目标表信息
    List<Target> targetDOList = redundantTargetService.listByRedundantSourceId(redundantSync.getRedundantSourceId());
    if (CollectionUtil.isEmpty(targetDOList)){
      //没有目标表，更新状态设置为成功
        redundantUpdateDO.setStatus(RedundantUpdateStatusEnum.SUCCESS.getCode());
    }
    //原表最新数据
    Map<String, Object> objMap = redundantSync.getSourceMap();
    //对每张需要冗余更新的目标表进行操作
    for (Target target : targetDOList) {
        RedundantSyncLogDO logDO = getRedundantSyncLogDO(redundantSync, target);
        try {
          //更新SQL组装
            //查询数据组装
            String searchParams = target.getSearchParams();
            String searchStr = sqlAssemble(objMap, searchParams, AND);
            logDO.setSearchStr(searchStr);
            //更新数据组装
            String setParams = target.getSetParams();
            String setStr = sqlAssemble(objMap, setParams, COMMA);
            target.setSetParams(setStr);
            logDO.setSetStr(setStr);
            Long lastId = null;
          //记录时间
            StopWatch searchWatch = new StopWatch(target.getTableName());
            StopWatch syncWatch= new StopWatch(target.getTableName());
          //统计更新数量
            long total = 0L;
            while (true) {
              //通过searchStr查询目标表需要修改的IDS
                List<Long> updateIds = dynamicService.selectDynamic(target, lastId, searchStr, redundantSync.getPageSize(),searchWatch);
                if (CollectionUtil.isEmpty(updateIds)) {
                    break;
                }
                lastId = updateIds.get(updateIds.size() - 1);
                total = total + updateIds.size();
              //真正的同步更新逻辑
                this.sync(target, updateIds,syncWatch);
                if (updateIds.size() < redundantSync.getPageSize()) {
                    break;
                }
            }
            if (redundantUpdateDO.getStatus() == null || RedundantUpdateStatusEnum.SUCCESS.getCode().equals(redundantUpdateDO.getStatus())) {
                redundantUpdateDO.setStatus(RedundantUpdateStatusEnum.SUCCESS.getCode());
            }
            logDO.setSearchDate(searchWatch.getTotalTimeSeconds());
            logDO.setSyncDate(syncWatch.getTotalTimeSeconds());
            logDO.setTotalDate(logDO.getSyncDate() + logDO.getSearchDate());
            logDO.setTotal(total);
        } catch (Exception e) {
            e.printStackTrace();
          //异常，FAIR(4,"同步失败");
            redundantUpdateDO.setStatus(RedundantUpdateStatusEnum.FAIR.getCode());
            logDO.setError(e.getMessage());
            logDO.setErrorDate(new Date());
        }
      //记录日志，redundant_sync_log
        redundantSyncLogService.save(logDO);
    }
      //更新redundant_update表
    redundantUpdateService.updateById(redundantUpdateDO);
}

    @Override
    public void sync(Target target, List<Long> idList,StopWatch stopWatch) {
        if (CollectionUtil.isEmpty(idList)) {
            return;
        }
      //更新目标表
        dynamicService.updateDynamic(target, idList,stopWatch);
    }

        UPDATE
            ${databaseName}.${tableName}
        SET
            ${setParams}
        WHERE id in
        <foreach collection="idList" separator="," open="(" index="index" item="id" close=")">
            #{id}
        </foreach>