php调用datashap,可解释机器学习的应用-shap值,shapvalue,使用

1 数据预处理和建模

1.1 加载库和数据预处理

import pandas as pd

import numpy as np

from sklearn.metrics import roc_auc_score, precision_recall_curve, roc_curve, average_precision_score

from sklearn.model_selection import KFold, train_test_split

from lightgbm import LGBMClassifier

import matplotlib.pyplot as pl

import gc # 对已经销毁的对象,Python不会自动释放其占据的内存空间。为了能够充分地利用分配的内存,避免程序跑到一半停止,要时不时地进行内存回收

import shap

file_path = 'D:\\jupyter files\\shap_value_practice_data\\home-credit-default-risk\\'

def build_model_input():

buro_bal = pd.read_csv(file_path + 'bureau_balance.csv')

print('Buro bal shape : ', buro_bal.shape)

print('transform to dummies')

buro_bal = pd.concat([buro_bal, pd.get_dummies(buro_bal.STATUS, prefix='buro_bal_status')], axis=1).drop('STATUS', axis=1)

print('Counting buros')

buro_counts = buro_bal[['SK_ID_BUREAU', 'MONTHS_BALANCE']].groupby('SK_ID_BUREAU').count()

buro_bal['buro_count'] = buro_bal['SK_ID_BUREAU'].map(buro_counts['MONTHS_BALANCE'])

print('averaging buro bal')

avg_buro_bal = buro_bal.groupby('SK_ID_BUREAU').mean()

avg_buro_bal.columns = ['avg_buro_' + f_ for f_ in avg_buro_bal.columns]

del buro_bal

gc.collect()

print('Read Bureau')

buro = pd.read_csv(file_path + 'bureau.csv')

print('Go to dummies')

buro_credit_active_dum = pd.get_dummies(buro.CREDIT_ACTIVE, prefix='ca_')

buro_credit_currency_dum = pd.get_dummies(buro.CREDIT_CURRENCY, prefix='cu_')

buro_credit_type_dum = pd.get_dummies(buro.CREDIT_TYPE, prefix='ty_')

buro_full = pd.concat([buro, buro_credit_active_dum, buro_credit_currency_dum, buro_credit_type_dum], axis=1)

# buro_full.columns = ['buro_' + f_ for f_ in buro_full.columns]

del buro_credit_active_dum, buro_credit_currency_dum, buro_credit_type_dum

gc.collect()

print('Merge with buro avg')

buro_full = buro_full.merge(right=avg_buro_bal.reset_index(), how='left', on='SK_ID_BUREAU', suffixes=('', '_bur_bal'))

print('Counting buro per SK_ID_CURR')

nb_bureau_per_curr = buro_full[['SK_ID_CURR', 'SK_ID_BUREAU']].groupby('SK_ID_CURR').count()

buro_full['SK_ID_BUREAU'] = buro_full['SK_ID_CURR'].map(nb_bureau_per_curr['SK_ID_BUREAU'])

print('Averaging bureau')

avg_buro = buro_full.groupby('SK_ID_CURR').mean()

print(avg_buro.head())

del buro, buro_full

gc.collect()

print('Read prev')

prev = pd.read_csv(file_path + 'previous_application.csv')

prev_cat_features = [

f_ for f_ in prev.columns if prev[f_].dtype == 'object'

]

print('Go to dummies')

prev_dum = pd.DataFrame()

for f_ in prev