ETL任务发散了

1、ODS 

2、DWD

insert into  xa_dwd_event_all_pt partition(pt)
select 
appversion,sessionId,tenantId,eventkey,eventtime,eventvars,pin,platform,
from_unixtime(cast(eventtime/1000 as bigint)) stime,
cast(from_unixtime(cast(eventtime/1000 as bigint)) as date) sdate,
get_json_object(eventvars, '$.imei') imei,
get_json_object(eventvars, '$.car_id') car_id,
get_json_object(eventvars, '$.source') source,
get_json_object(eventvars, '$.result') result,
get_json_object(eventvars, '$.fail_reason') fail_reason,
get_json_object(eventvars, '$.service_id') service_id,
get_json_object(eventvars, '$.end_time') end_time,
get_json_object(eventvars, '$.firstLevelMenu') firstLevelMenu,
get_json_object(eventvars, '$.secondLevelMenu') secondLevelMenu,
get_json_object(eventvars, '$.riding_time') riding_time,
get_json_object(eventvars, '$.riding_distance') riding_distance,
get_json_object(eventvars, '$.platform') secondLevelMenu,
get_json_object(eventvars, '$.app_version') app_version,
get_json_object(eventvars, '$.order_id') order_id,
ROW_NUMBER() over (partition by tenantId,eventkey,pin,sessionId order by eventtime) rn,
pt
from xa_ods_log_event_new_pt 
-- where  cast(from_unixtime(cast(eventtime/1000 as bigint)) as date)= DATE_SUB(now(),1);
  where pt = to_char(date_sub(now(),1),'yyyymmdd');

insert into 
xa_dwd_log_one_click_return
-- create table xa_dwd_log_one_click_return as 
select sdate,tenantId,service_id,count(*) one_click_num  
from   xa_dwd_event_all_pt
where 
result='success'  
and eventkey= 'end_ebike'  
and platform='物理一键还车'
and pt = to_char(date_sub(now(),1),'yyyymmdd')
group by sdate,tenantId,service_id;

发现指标异常，定位到时该etl任务问题；

pt为业务同步日期分区字段，sdate为sls日志时间戳解析字段;

问题：

SLS日志

${bizdate}000000 与eventtime 不一致，ods层进入了脏数据；

eventtime为20230904的数据在pt=20230905中出现了，导致后续表数据发散；

解决：

历史

临时修复

 过滤修复sdate = date_sub(now(),1)