import sys
print("code_path:",*sys.argv)
spark.sparkContext.setLogLevel("Error")
#通过调脚本时传递的参数,定义日期
today = datetime.datetime.today() if len(sys.argv) < 3 else
datetime.datetime.strptime.(sys.argv[2],"%Y%m%d")
this_year = today.year
yesterday = today + timedelta(days=-1)
yesterday_format = yesterday.strftime("%Y%m%d")
before_yesterday = today + timedelta(days=-2)
before_yesterday_format = befor_yesterday.strftime("%Y%m%d")
3_days_ago = (today + timedelta(days=-3)).strftime("%Y%m%d")
365_days_ago = (yesterday + timedelta(days=-365)).strftime("%Y%m%d")
banks_info = spark.sql("""select bank,version,database,clm_database
from sys_bank_database_version """).join(spark.sql(""" select from sys_parameter
lateral view explode(split(parameter_value,',')) t as bank
"""),"bank","inner").collect()
print('\n',banks_info,'\n')
for i in range(len(banks_info)):
bank,db_version,database,clm_database = banks_info[i].bank,banks_info[i].version,
banks_info[i].database.lower(),banks_info[i].clm_database
print('\n',bank,db_version,database,clm_database)
try:
exist_flag = spark.read.format("csv").load(f"/user/rongshu/mapp/mallpointcashparam/{bank}/{3_days_ago}")
.count()
except:
exist_flag = False
if exist_flag:
begin_date = befor_yesterday_format
else:
begin_date = 365_days_ago
print(begin_date,end_date,f" EXIST PREVIOUS DATA:{exist_flag}")
spark.table("t_order_product").persist()
spark.table("t_order_product").unpersist()
04-16
5256