def data_check_distinct(rows, distinct_rule, table_name): # 获取全部重复的数据 duplicate_bool_all = rows.duplicated(subset=distinct_rule, keep=False) # 获取重复的第一行 duplicate_bool_first = rows.duplicated(subset=distinct_rule, keep='first') repeat_all = rows.loc[duplicate_bool_all == True] repeat_first = rows.loc[duplicate_bool_first == False] df = pd.DataFrame() df2 = pd.DataFrame() df2['id'] = repeat_first.iloc[:, 0] df2['is_initialize'] = 'TRUE' s = df2.set_index('id')['is_initialize'] df['id'] = repeat_all.iloc[:, 0] df['is_initialize'] = '' df['table_name'] = table_name df['is_initialize'] = df['id'].map(s).fillna(df['is_initialize']).astype(str) df['creattime'] = time_nyr(None) df.to_sql('repeating_data', con=engine(), index=False, if_exists='append', chunksize=10000)
rows是原始数据,
distinct_rule是字典校验是指的哪些字段判断重复distinct_rule =['xm','zjh','xb','csrq']
table_name是保存的表面