由于每次迭代都需要使用同一个数据集,只是label不一样,考虑将label取出来后进行随机打散,然后使用Dataset的set_label函数重新设置标签。从而优化数据集的创建环节。
from sklearn.metrics import *
from tqdm import tqdm
import lightgbm as lgb
def get_feature_importances_by_dtrain(dtrain, y_, train_features, shuffle=False,categorical_feats=[], seed=2024):
if shuffle:
# Here you could as well use a binomial distribution
# y = data['TARGET'].copy().sample(frac=1.0)
# 复用数据dtrain,从而优化数据构建时间
y = y_.copy().sample(frac=1.0)
# y = dtrain.get_label().copy().sample(frac=1.0)
dtrain.set_label(y)
else:
y = y_
# Fit LightGBM in RF mode, yes it's quicker than sklearn RandomForest
lgb_params = {
'objective': 'binary',
'boosting_type': 'rf',
'subsample': 0.623,
'colsample_bytree': 0.7,
'num_leaves': 127,
'max_depth': 7,
'seed': seed,
'bagging_freq': 5,
'verbose': -1,
'n_jobs': 80,
# 'device': 'gpu',
# 'gpu_platform_id': 0,
# 'gpu_device_id': 0
}
# Fit the model
clf = lgb.train(params=lgb_params, train_set=dtrain, num_boost_round=200, categorical_feature=categorical_feats)
# Get feature importances
imp_df = pd.DataFrame()
imp_df["feature"] = list(train_features)
imp_df["importance_gain"] = clf.feature_importance(importance_type='gain')
imp_df["importance_split"] = clf.feature_importance(importance_type='split')
#imp_df['trn_score'] = roc_auc_score(y, clf.predict(data[train_features]))
imp_df['trn_score'] = roc_auc_score(y, clf.predict(dtrain.data))
return imp_df
%%time
print('start')
label = 'label7d'
# &(train_data2['platformtype']=='ANDROID')\
# &(train_data2['terms']!=2)\
# &(train_data2['term_time_period']<=15)\
# &(train_data2['label0d']==0) # 限制当日未完件
# &(train_data2['is_0d_curp_reject']==0) # 限制当日未完件
data = data_all_feature_df[(data_all_feature_df.platform_type_name.isin([ 'ANDROID']))
&(data_all_feature_df.sdk_type=='M')
&(data_all_feature_df.register_month<='2024-01')
&(data_all_feature_df['label0d']==0) #
&(data_all_feature_df['is_0d_curp_reject']==0) #
][[label]+select_columns].sample(200000)
y = data[label]
print(len(y))
dtrain = lgb.Dataset(data[select_columns]
, y
, free_raw_data=False)
print('dtrain end')
print(dtrain.data.shape)
%%time
actual_imp_df = get_feature_importances_by_dtrain(dtrain, y, train_features=select_columns, shuffle=False, categorical_feats=[], seed=2024)
print('actual_imp_df end')
%%time
null_imp_df = pd.DataFrame()
nb_runs = 80
for i in tqdm(range(nb_runs)):
# Get current run importances
imp_df = get_feature_importances_by_dtrain(dtrain, y, train_features=select_columns, shuffle=True, categorical_feats=[], seed=2024)
# Concat the latest importances with the old ones
null_imp_df = pd.concat([null_imp_df, imp_df], axis=0)
print('null_imp_df end')
feature_scores = []
for _f in actual_imp_df['feature'].unique():
f_null_imps_gain = null_imp_df.loc[null_imp_df['feature'] == _f, 'importance_gain'].values
f_act_imps_gain = actual_imp_df.loc[actual_imp_df['feature'] == _f, 'importance_gain'].mean()
gain_score = np.log(1e-10 + f_act_imps_gain / (1 + np.percentile(f_null_imps_gain, 75))) # Avoid didvide by zero
f_null_imps_split = null_imp_df.loc[null_imp_df['feature'] == _f, 'importance_split'].values
f_act_imps_split = actual_imp_df.loc[actual_imp_df['feature'] == _f, 'importance_split'].mean()
split_score = np.log(1e-10 + f_act_imps_split / (1 + np.percentile(f_null_imps_split, 75))) # Avoid didvide by zero
feature_scores.append((_f, split_score, gain_score))
scores_df = pd.DataFrame(feature_scores, columns=['feature', 'split_score', 'gain_score'])
scores_df = scores_df.sort_values('split_score', ascending=False)
actual_imp_df.to_pickle(f'cuwanjian_actual_imp_df_{data_version}_{label_dt}.pkl')
null_imp_df.to_pickle(f'cuwanjian_null_imp_df_{data_version}_{label_dt}.pkl')
scores_df.to_pickle(f'cuwanjian_feature_scores_df_{data_version}_{label_dt}.pkl')