1. 设计目的
公司系统与第三方同步接口调用频繁出错,因此业务提出需求,需要在本系统中第三方调用失败的接口,能够看到重试日志,错误原因,当错误原因解决后,能够发起重试请求,对第三方接口调用失败的请求进行重发。
2. 技术实现方案
首先, 通过spring 切面拦截到第三方请求,获取到请求头部、请求参数、请求体等相关请求信息,生成一张同步请求日志表,将这些请求数据放入到数据库表中,然后运行定时任务,定时扫描
同步请求日志表,获取到同步失败的数据,将失败的数据放入到同步重试日志表中,前端获取到这些同步失败的数据,将数据中包含的请求方式、请求头、请求参数、请求体进行封装,点击重试按钮,将封装的请求进行重新调用,后端再将重新调用的返回结果更新重试日志表中。
注意: 前端重发的请求将会重新携带以下两个参数,retry 标识是重试请求, businessId 标识重试的是那条数据,这样做的目的是重试请求返回结果会更改重试日志表的返回结果,同时重试请求不会再进入同步日志表,将两张表的功能区分开来。
扩展: 目前数据量不大,先使用数据库做存储,如果以后数据量大了,可以把数据库替换为mongdb 或者 es,对代码侵入性也不强,比较方便扩展。
3. 代码实现细节
3.1 请求拦截切面类 SyncLogAspect
/**
* @author tuwenbin@newhope.cn
* @Title: SyncLogAspect
* @Description: 拦截同步请求, 记录同步日志
* @date 2021/11/8 17:50
*/
@Aspect
@Component
@Slf4j
@Order(1)
public class SyncLogAspect {
@Autowired
private PluginSyncLogService syncLogService;
@Autowired
private PluginRetryLogService retryLogService;
@Autowired
private PluginApiService pluginApiService;
@Pointcut("@annotation(GenerateSyncLog)|| @within(GenerateSyncLog)")
public void syncLogPointcut() {
}
@Around("syncLogPointcut()")
public Object doGenerateSyncLog(ProceedingJoinPoint joinPoint) throws Throwable {
// 获取HttpServletRequest
ServletRequestAttributes attributes = (ServletRequestAttributes) RequestContextHolder.getRequestAttributes();
HttpServletRequest request = attributes.getRequest();
String requestURL= request.getRequestURI();
//获取url格式 /plugin-api/api-boh-v1/pushMaterialInfo
String[] urlArrays = requestURL.split(PluginConstant.SPLIT_INCLINED_ROD);
if(urlArrays.length >= PluginConstant.URL_LENGTH){
String pluginName = urlArrays[2];
String method = urlArrays[3];
if(LogConfigEnum.containMethod(pluginName, method)){
//判断是否是重试请求, 如果是重试请求, 直接将请求放在重试日志表中
String retry = request.getParameter(PluginConstant.REQUEST_RETRY);
String businessId = request.getParameter(PluginConstant.REQUEST_BUSINESSID);
PluginSyncLog syncLog = new PluginSyncLog();
if(!PluginConstant.RETRY_FLAG.equals(retry)){
syncLog = buildSyncLog(request, pluginName, method);
}
Object proceed = joinPoint.proceed();
if(PluginConstant.RETRY_FLAG.equals(retry)){
updateRetryLog(pluginName, method, proceed, businessId);
}else {
String syncStatus = getSyncStatus(proceed, pluginName, method);
syncLog.setStatus(syncStatus);
syncLog.setResponseData(JSONObject.toJSONString(proceed));
syncLogService.save(syncLog);
}
return proceed;
}
}
return joinPoint.proceed();
}
/**
* 更新重试日志
* @param pluginName
* @param method
* @param proceed
* @param businessId
*/
private void updateRetryLog(String pluginName, String method, Object proceed, String businessId) {
PluginRetryLog retryLog = retryLogService.getById(Long.valueOf(businessId));
if(Objects.nonNull(retryLog)){
LogHandler logHandler = RetryLogStrategyFactory.getLogHandler(pluginName, method);
RetryResult retryResult = logHandler.methodExecute(proceed);
retryLog.setRetryStatus(retryResult.getRetryStatus());
retryLog.setFailureReason(retryResult.getFailureReason());
retryLogService.saveOrUpdate(retryLog);
}
}
/**
* 获取同步状态, 返回结果list中有未成功的,记录同步标识为失败
* @param result
* @param pluginName
* @param method
*/
private String getSyncStatus(Object result, String pluginName, String method) {
LogHandler logHandler = RetryLogStrategyFactory.getLogHandler(pluginName, method);
if(Objects.isNull(logHandler)){
throw new BizException("未配置对应的日志处理器");
}
String syncStatus = logHandler.getSyncStatus(result);
return syncStatus;
}
private PluginSyncLog buildSyncLog(HttpServletRequest request, String pluginName, String method) {
PluginSyncLog syncLog = new PluginSyncLog();
syncLog.setRequestUrl(request.getRequestURI());
syncLog.setRequestHeader(JSONArray.toJSONString(getHeadersInfo(request)));
syncLog.setRequestMethod(request.getMethod());
syncLog.setRequestParams(request.getQueryString());
LogConfigEnum configEnum = LogConfigEnum.valueOf(pluginName, method);
syncLog.setDataSource(configEnum.getDataSource().getSource());
syncLog.setDataTarget(configEnum.getDataTarget());
syncLog.setDataType(configEnum.getDataType().getType());
String body = ((RequestWrapper) request).getBody();
syncLog.setSyncTime(LocalDateTime.now());
syncLog.setRequestBody(body);
//设置推送目的地
PushTargetVo pushTargetVo = pluginApiService.getTenantIdAndOrgId(request, pluginName, syncLog.getDataSource());
syncLog.setTenantId(pushTargetVo.getTenantId());
syncLog.setOrgId(pushTargetVo.getOrgId());
return syncLog;
}
private Map<String, String> getHeadersInfo(HttpServletRequest request) {
Map<String, String> map = new HashMap<>();
Enumeration headerNames = request.getHeaderNames();
while (headerNames.hasMoreElements()) {
String key = (String) headerNames.nextElement();
String value = request.getHeader(key);
map.put(key, value);
}
return map;
}
}
3.2 定时任务扫描 GenerateRetryLogJob
@Component
@Slf4j
public class GenerateRetryLogJob {
@Autowired
private PluginSyncLogService syncLogService;
@Autowired
private PluginRetryLogService retryLogService;
/**
* 生成重试错误日志, 每5分钟执行一次
*/
@Scheduled(cron = "0 */1 * * * ?")
public void generateRetryLog() {
String date = DateUtils.getDate();
LocalDateTime now = LocalDateTime.now();
String lastWeek = DateUtils.formatDate(now.minusDays(7), null);
List<PluginRetryLog> retryLogList = new ArrayList<>();
List<PluginSyncLogVo> failSyncLogList = syncLogService.findFailSyncLogList(lastWeek, date);
if(!CollectionUtils.isEmpty(failSyncLogList)){
failSyncLogList.forEach(logVo -> {
String dataSource = logVo.getDataSource();
String dataType = logVo.getDataType();
String responseData = logVo.getResponseData();
LogConfigEnum logConfigEnum = LogConfigEnum.configEnumOf(dataSource, dataType);
LogHandler logHandler = RetryLogStrategyFactory.getLogHandler(logConfigEnum.getPluginName(), logConfigEnum.getMethod());
//获取到失败的业务code集合
Map<String, ResultData> failMap = logHandler.getFailMap(responseData);
List<PluginRetryLog> retryLogs = logHandler.parseRequestBody(logVo, failMap, responseData);
retryLogList.addAll(retryLogs);
});
}
if(!CollectionUtils.isEmpty(retryLogList)){
retryLogService.saveBatch(retryLogList);
}
if(!CollectionUtils.isEmpty(failSyncLogList)){
List<PluginSyncLog> syncLogs = new ArrayList<>();
failSyncLogList.forEach(failSyncLog ->{
PluginSyncLog syncLog = syncLogService.getById(failSyncLog.getId());
syncLog.setRetryFlag(PluginConstant.RETRY_FLAG_TRUE);
syncLogs.add(syncLog);
});
syncLogService.saveOrUpdateBatch(syncLogs);
}
}
}
3.3 日志处理器 LogHandler
在系统与第三方请求对接中,第三方有多个,且由不同的人员对接开发完成,没有设计统一的请求与响应对象,为了对这种情况进行兼容,设计了日志处理器 LogHandler,运用工厂模式与策略模式,实现了对不同的第三方,不同的请求url,可以实现自己的日志处理器,进行灵活的数据处理。
同时当调用某个接口批量传输数据,数据可能有的成功,有的失败,且数据的失败原因不一样,为了兼容这种情况,运用处理器可以对数据进行拆分,拆分为单条,放入到重试日志表中,进行重试。
/**
* @author tuwenbin@newhope.cn
* @Title: Loghandler
* @Description: 日志处理器
* <R> R 返回类型
* @date 2021/11/11 11:03
*/
public interface LogHandler<R> {
/**
* 获取数据同步状态
* @param result
* @return
*/
String getSyncStatus(R result);
/**
* 判断是否全局错误
* @param responseData
* @return
*/
boolean isGlobalError(String responseData);
/**
* 解析生成重试方法执行结果
* @param result
* @return
*/
RetryResult methodExecute(R result);
/**
* 获取到失败集合
* @param responseData 相应内容
* @return
*/
Map<String, ResultData> getFailMap(String responseData);
/**
* 解析请求数据, 构建重试日志集合
* @param logVo 失败数据
* @param failMap 失败集合
* @param responseData 相应数据
* @return
*/
List<PluginRetryLog> parseRequestBody(PluginSyncLogVo logVo, Map<String, ResultData> failMap, String responseData);
处理器工厂类
public class RetryLogStrategyFactory {
private static Map<LogConfigEnum, LogHandler> logHandlerMap = new HashMap<>();
static {
logHandlerMap.put(LogConfigEnum.YX_PUSH_ORDER, new YxOrderLogHandler());
logHandlerMap.put(LogConfigEnum.YX_PUSH_MATERIAL, new YxMaterialLogHandler());
logHandlerMap.put(LogConfigEnum.YX_PUSH_STORE, new YxStoreLogHandler());
logHandlerMap.put(LogConfigEnum.WG_PUSH_ORDER, new WgLogHandler());
logHandlerMap.put(LogConfigEnum.WG_PUSH_MATERIAL, new WgLogHandler());
}
/**
* 获取日志处理器
* @param pluginName 插件名称
* @param method 调用方法
* @return
*/
public static LogHandler getLogHandler(String pluginName, String method){
LogConfigEnum configEnum = LogConfigEnum.valueOf(pluginName, method);
LogHandler logHandler = logHandlerMap.get(configEnum);
return logHandler;
}
}
4. 扩展示例
4.1 新增配置项
当我们新增一个第三方接口调用来源时, 在枚举类 DataSourceEnum 类中新增一个来源
当我们新增一个请求时,在 LogConfigEnum 中新增一个请求类型、请求配置项,设置好对应的请求 uri。
4.2 新增日志处理器
可以对这个请求来源先实现一个公共的请求处理,实现一些公共的方法。
/**
* @author tuwenbin@newhope.cn
* @Title: YxLogHandler
* @Description: yx 调用处理
* @date 2021/11/12 10:01
*/
public class YxLogHandler implements LogHandler<YunXianRespResult>{
@Override
public String getSyncStatus(YunXianRespResult result) {
String status = PluginConstant.SUCCESS_CODE;
if(!PluginConstant.SUCCESS_CODE.equals(result.getCode())){
status = PluginConstant.FAIL_CODE;
}
List<ResultData> dataList = result.getData();
if(!CollectionUtils.isEmpty(dataList)){
for(ResultData data : dataList){
String code = data.getCode();
if(!PluginConstant.SUCCESS_CODE.equals(code)){
status = PluginConstant.FAIL_CODE;
}
}
}
return status;
}
@Override
public boolean isGlobalError(String responseData) {
YunXianRespResult result = JSONObject.parseObject(responseData, YunXianRespResult.class);
return !PluginConstant.SUCCESS_CODE.equals(result.getCode());
}
@Override
public RetryResult methodExecute(YunXianRespResult result) {
RetryResult retryResult = new RetryResult();
Integer retryStatus = RetryStatusEnum.FAIL.getStatus();
List<ResultData> dataList = result.getData();
if(!CollectionUtils.isEmpty(dataList)){
//数据会被拆分,只有一条数据
ResultData resultData = dataList.get(0);
if(PluginConstant.SUCCESS_CODE.equals(resultData.getCode())){
retryStatus = RetryStatusEnum.SUCCESS.getStatus();
}else {
retryResult.setFailureReason(resultData.getErrorMsg());
}
retryResult.setBusinessCode(resultData.getBusinessCode());
}else {
String code = result.getCode();
if(PluginConstant.SUCCESS_CODE.equals(code)){
retryStatus = RetryStatusEnum.SUCCESS.getStatus();
}
retryResult.setFailureReason(result.getErrorMsg());
}
retryResult.setRetryStatus(retryStatus);
return retryResult;
}
@Override
public Map<String, ResultData> getFailMap(String responseData) {
Map<String, ResultData> failMap = new HashMap<>(16);
YunXianRespResult respResult = JSONObject.parseObject(responseData, YunXianRespResult.class);
List<ResultData> dataList = JSONArray.parseArray(JSONArray.toJSONString(respResult.getData()), ResultData.class);
if(!CollectionUtils.isEmpty(dataList)){
dataList.forEach(data -> {
String code = data.getCode();
if(!PluginConstant.SUCCESS_CODE.equals(code)){
failMap.put(data.getBusinessCode(), data);
}
});
}
return failMap;
}
@Override
public List<PluginRetryLog> parseRequestBody(PluginSyncLogVo logVo, Map<String, ResultData> failMap, String responseData) {
return null;
}
}
然后对每个请求需求额外处理,不一样的,再实现单独的请求处理器。
public class YxMaterialLogHandler extends YxLogHandler {
@Override
public List<PluginRetryLog> parseRequestBody(PluginSyncLogVo logVo, Map<String, ResultData> failMap, String responseData) {
List<PluginRetryLog> retryLogList = new ArrayList<>();
List<PushMaterialInfoReq> reqList = JSONArray.parseArray(logVo.getRequestBody(), PushMaterialInfoReq.class);
if(this.isGlobalError(responseData)){
YunXianRespResult result = JSONObject.parseObject(responseData, YunXianRespResult.class);
PluginRetryLog retryLog = PluginUtil.buildGlobalRetryLog(logVo, result.getErrorMsg());
retryLogList.add(retryLog);
}else {
if(!CollectionUtils.isEmpty(reqList)){
reqList.forEach(req -> {
String goodCode = req.getMerchandiseCode();
if(failMap.containsKey(goodCode)){
List<PushMaterialInfoReq> requestList = new ArrayList<>();
requestList.add(req);
ResultData failData = failMap.get(goodCode);
PluginRetryLog retryLog = PluginUtil.buildRetryLog(logVo, failData);
retryLog.setRequestBody(JSONArray.toJSONString(requestList));
retryLogList.add(retryLog);
}
});
}
}
return retryLogList;
}
}
public class YxOrderLogHandler extends YxLogHandler{
@Override
public List<PluginRetryLog> parseRequestBody(PluginSyncLogVo logVo, Map<String, ResultData> failMap, String responseData) {
List<PluginRetryLog> retryLogList = new ArrayList<>();
List<PushOrderInfoReq> reqList = JSONArray.parseArray(logVo.getRequestBody(), PushOrderInfoReq.class);
if(this.isGlobalError(responseData)){
YunXianRespResult result = JSONObject.parseObject(responseData, YunXianRespResult.class);
PluginRetryLog retryLog = PluginUtil.buildGlobalRetryLog(logVo, result.getErrorMsg());
retryLogList.add(retryLog);
}else {
if(!CollectionUtils.isEmpty(reqList)){
reqList.forEach(req -> {
String orderCode = req.getOrderCode();
if(failMap.containsKey(orderCode)){
List<PushOrderInfoReq> requestList = new ArrayList<>();
requestList.add(req);
ResultData failData = failMap.get(orderCode);
PluginRetryLog retryLog = PluginUtil.buildRetryLog(logVo, failData);
retryLog.setRequestBody(JSONArray.toJSONString(requestList));
retryLogList.add(retryLog);
}
});
}
}
return retryLogList;
}
}
4.3 重试请求关闭签名校验