信用卡分级Spark,hadoop

import warnings

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
plt.rcParams['font.sans-serif']=['SimHei']
plt.rcParams['axes.unicode_minus']=False
warnings.filterwarnings("ignore")

pd.set_option('display.max_columns',1000)
pd.set_option('display.width', 1000)
pd.set_option('display.max_colwidth',1000)
data=pd.read_csv('D:/database/training.csv')
data.drop(data.iloc[:,:1],inplace = True,axis=1)
data.info()
print(data.isnull().sum())
data = data.dropna(subset=["NumberOfDependents"])
data.info()
# 用随机森林对缺失值预测填充函数

from sklearn.ensemble import RandomForestRegressor


def set_missing(df):
    process_df = df.iloc[:,[5,0,1,2,3,4,6,7,8,9]] # 把已有的数值型特征取出来
    # 分成已知该特征和未知该特征两部分
    known = process_df[process_df.MonthlyIncome.notnull()].iloc[:,:].values
    unknown = process_df[process_df.MonthlyIncome.isnull()].iloc[:,:].values
    X = known[:, 1:]    # X为特征属性值
    y = known[:, 0]    # y为结果标签值
    rfr = RandomForestRegressor(random_state=0, n_estimators=200,max_depth=3,n_jobs=-1)
    rfr.fit(X,y)
    # 用得到的模型进行未知特征值预测 月收入
    predicted = rfr.predict(unknown[:, 1:]).round(0)
    print(predicted)
    # 用得到的预测结果填补原缺失数据
    df.loc[df.MonthlyIncome.isnull(), 'MonthlyIncome'] = predicted
    return df
data=set_missing(data)#用随机森林填补比较多的缺失值
data = data.drop_duplicates()#删除重复项
data.info()
print(data.describe())



# 首先，我们发现变量age中存在0，显然是异常值，直接剔除：

# 年龄等于0的异常值进行剔除
data = data[data['age'] > 0]
# 对于变量NumberOfTime30-59DaysPastDueNotWorse、NumberOfTimes90DaysLate、NumberOfTime60-89DaysPastDueNotWorse这三个变量，
# 由下面的箱线图图3-2可以看出，均存在异常值，且由unique函数可以得知均存在96、98两个异常值，因此予以剔除。
# 同时会发现剔除其中一个变量的96、98值，其他变量的96、98两个值也会相应被剔除。

plt.figure(figsize=(16,12))

fig, axes = plt.subplots(1, 3)


datatemp1 = data[
    ["NumberOfTime30-59DaysPastDueNotWorse", "NumberOfTimes90DaysLate", "NumberOfTime60-89DaysPastDueNotWorse"]]
datatemp1.plot(kind='box', ax=axes, subplots=True)
# sym参数表示异常值标记的方式

axes[0].set_ylabel('30-59')
axes[1].set_ylabel('>=90')
axes[2].set_ylabel('60-89')

fig.subplots_adjust(wspace=3, hspace=1)
plt.show()
data = data[data['NumberOfTime30-59DaysPastDueNotWorse']<90]
data = data[data['NumberOfTimes90DaysLate']<90]
data = data[data['NumberOfTime60-89DaysPastDueNotWorse']<90]
data = data.reset_index(drop=True)

data = data[data['age'] > 0]
age=data['age']
sns.distplot(age)
plt.show()
#月收入大于100000的有70个数据，我们将其取代为100000，不妨碍他们仍然为高收入群体
print(data[data['MonthlyIncome'] > 100000].count())
data["MonthlyIncome"][data["MonthlyIncome"]>100000]= 100000
mi=data['MonthlyIncome']
sns.distplot(mi)
plt.show()

corr = data.corr()
fig = plt.figure(figsize = (15,10))
ax1 = fig.add_subplot(1, 1, 1)
sns.heatmap(corr, annot=True, cmap='rainbow', ax=ax1, annot_kws={'size': 9, 'weight': 'bold', 'color': 'blue'})#绘制相关性系数热力图
plt.show()


from sklearn.model_selection import train_test_split
Y = data['SeriousDlqin2yrs']
X=data.iloc[:,1:]
X_train, X_test, Y_train, Y_test = train_test_split(X,Y,train_size = 0.8,random_state=0)
train = pd.concat([Y_train,X_train], axis =1)
test = pd.concat([Y_test,X_test], axis =1)
train = train.reset_index(drop=True)
test = test.reset_index(drop=True)
#保留一份测试数据集，后面生成评分卡
test.to_csv('origin_test.csv', index=False)

import scipy.stats as stats
def mono_bin(Y, X, n = 20):
    r = 0
    good=Y.sum()
    bad=Y.count()-good
    while np.abs(r) < 1:
        d1 = pd.DataFrame({"X": X, "Y": Y, "Bucket": pd.qcut(X, n,duplicates="drop")})
        # 后面报错You can drop duplicate edges by setting the 'duplicates' kwarg，所以回到这里补充duplicates参数
        # pandas中使用qcut()，边界易出现重复值，如果为了删除重复值设置 duplicates=‘drop’，则易出现于分片个数少于指定个数的问题
        # 经尝试，设置duplicates参数为“drop”可行，而不能设置为“raise”。
        d2 = d1.groupby('Bucket', as_index = True)
        r, p = stats.spearmanr(d2.mean().X, d2.mean().Y)
        n = n - 1
    d3 = pd.DataFrame(d2.X.min(), columns = ['min'])
    d3['min']=d2.min().X
    d3['max'] = d2.max().X
    d3['sum'] = d2.sum().Y
    d3['total'] = d2.count().Y
    d3['rate'] = d2.mean().Y
    d3['woe']=np.log((d3['rate']/(1-d3['rate']))/(good/bad))
    d4 = (d3.sort_values(by = 'min')).reset_index(drop=True)
    print("=" * 60)
    print(d4)
    woe=list(d4['woe'].round(3))
    return d4
    print(d4)



data=data.drop_duplicates(subset=None,keep='first',inplace=False)
data.shape
mono_bin(data.SeriousDlqin2yrs,data.RevolvingUtilizationOfUnsecuredLines)
mono_bin(data.SeriousDlqin2yrs,data.age)
mono_bin(data.SeriousDlqin2yrs,data.MonthlyIncome)

pinf = float('inf')#正无穷大
ninf = float('-inf')#负无穷大

cutx3 = [ninf, 0, 1, 3, 5, pinf]
cutx6 = [ninf, 1, 2, 3, 5, pinf]
cutx7 = [ninf, 0, 1, 3, 5, pinf]
cutx8 = [ninf, 0,1,2, 3, pinf]
cutx9 = [ninf, 0, 1, 3, pinf]
cutx10 = [ninf, 0, 1, 2, 3, 5, pinf]


def monoto_bin(Y, X, n):
    r = 0
    total_good = Y.sum()
    total_bad =Y.count()-total_good
    while np.abs(r) < 1:
        d1 = pd.DataFrame({"X": X, "Y": Y, "Bucket": pd.qcut(X, n)})
        d2 = d1.groupby('Bucket', as_index = True)
        r, p = stats.spearmanr(d2.mean().X, d2.mean().Y)
        n = n - 1
    d3 = pd.DataFrame(d2.min().X, columns = ['min_' + X.name])
    d3['min_' + X.name] = d2.min().X
    d3['max_' + X.name] = d2.max().X
    d3[Y.name] = d2.sum().Y
    d3['total'] = d2.count().Y
    #d3[Y.name + '_rate'] = d2.mean().Y
    #好坏比，求woe,证据权重，自变量对目标变量有没有影响，什么影响
    d3['goodattr']=d3[Y.name]/total_good
    d3['badattr']=(d3['total']-d3[Y.name])/total_bad
    d3['woe'] = np.log(d3['goodattr']/d3['badattr'])
    #iv，信息值，自变量对于目标变量的影响程度
    iv = ((d3['goodattr']-d3['badattr'])*d3['woe']).sum()
    d4 = (d3.sort_values(by = 'min_' + X.name)).reset_index(drop = True)
    print ("=" * 80)
    print (d4)
    cut = []
    cut.append(float('-inf'))
    for i in range(1,n+1):
        qua =X.quantile(i/(n+1))
        cut.append(round(qua,4))
    cut.append(float('inf'))
    woe = list(d4['woe'].round(3))
    return d4,iv,cut,woe

dfx1,ivx1,cutx1,woex1 = monoto_bin(data['SeriousDlqin2yrs'],data['RevolvingUtilizationOfUnsecuredLines'],n = 10)
dfx2,ivx2,cutx2,woex2 = monoto_bin(data['SeriousDlqin2yrs'],data['age'],n = 10)
dfx4,ivx4,cutx4,woex4 = monoto_bin(data['SeriousDlqin2yrs'],data['DebtRatio'],n = 10)



import scipy.stats as stats

def self_bin(Y,X,cat):
    good=Y.sum()
    bad=Y.count()-good
    d1=pd.DataFrame({'X':X,'Y':Y,'Bucket':pd.cut(X,cat)})
    d2=d1.groupby('Bucket', as_index = True)
    d3 = pd.DataFrame(d2.X.min(), columns=['min'])
    d3['min'] = d2.min().X
    d3['max'] = d2.max().X
    d3['sum'] = d2.sum().Y
    d3['total'] = d2.count().Y
    d3['rate'] = d2.mean().Y
    d3['woe'] = np.log((d3['rate'] / (1 - d3['rate'])) / (good / bad))
    d3['goodattribute'] = d3['sum'] / good
    d3['badattribute'] = (d3['total'] - d3['sum']) / bad
    iv = ((d3['goodattribute'] - d3['badattribute']) * d3['woe']).sum()
    d4 = (d3.sort_values(by='min'))
    print("=" * 60)
    print(d4)
    woe = list(d4['woe'].round(3))
    return d4, iv,woe
#对剩余变量进行自定义分箱

pinf = float('inf')#正无穷大
ninf = float('-inf')#负无穷大
cutx3 = [ninf, 0, 1, 3, 5, pinf]
cutx5 = [ninf,1000,2000,3000,4000,5000,6000,7500,9500,12000,pinf]
cutx6 = [ninf, 1, 2, 3, 5, pinf]
cutx7 = [ninf, 0, 1, 3, 5, pinf]
cutx8 = [ninf, 0,1,2, 3, pinf]
cutx9 = [ninf, 0, 1, 3, pinf]
cutx10 = [ninf, 0, 1, 2, 3, 5, pinf]
dfx3, ivx3,woex3 = self_bin(data['SeriousDlqin2yrs'],data['NumberOfTime30-59DaysPastDueNotWorse'],cutx3)
dfx5, ivx5,woex5 = self_bin(data['SeriousDlqin2yrs'],data['MonthlyIncome'],cutx5)
dfx6, ivx6,woex6 = self_bin(data['SeriousDlqin2yrs'],data['NumberOfOpenCreditLinesAndLoans'],cutx6)
dfx7, ivx7,woex7 = self_bin(data['SeriousDlqin2yrs'],data['NumberOfTimes90DaysLate'],cutx7)
dfx8, ivx8,woex8 = self_bin(data['SeriousDlqin2yrs'],data['NumberRealEstateLoansOrLines'],cutx8)
dfx9, ivx9,woex9 = self_bin(data['SeriousDlqin2yrs'],data['NumberOfTime60-89DaysPastDueNotWorse'],cutx9)
dfx10, ivx10,woex10 = self_bin(data['SeriousDlqin2yrs'],data['NumberOfDependents'],cutx10)

ivlist=[ivx1,ivx2,ivx3,ivx4,ivx5,ivx6,ivx7,ivx8,ivx9,ivx10]#各变量IV
index=['x1','x2','x3','x4','x5','x6','x7','x8','x9','x10']#x轴的标签
fig1 = plt.figure(1)
ax1 = fig1.add_subplot(1, 1, 1)
x = np.arange(len(index))+1 #设置x轴柱子的个数
ax1.bar(x, ivlist, width=0.4)#生成柱状图
ax1.set_xticks(x)  #设置x轴的刻度
ax1.set_xticklabels(index, rotation=0, fontsize=12)
ax1.set_ylabel('IV(Information Value)', fontsize=14)

#在柱状图上添加数字标签
for a, b in zip(x, ivlist):
    plt.text(a, b + 0.01, '%.4f' % b, ha = 'center', va = 'bottom', fontsize = 10)
plt.show()

#替换成woe函数
def trans_woe(var,var_name,woe,cut):
    woe_name=var_name+'_woe'
    for i in range(len(woe)):       # len(woe) 得到woe里 有多少个数值
        if i==0:
            var.loc[(var[var_name]<=cut[i+1]),woe_name]=woe[i]  #将woe的值按 cut分箱的下节点，顺序赋值给var的woe_name 列 ，分箱的第一段
        elif (i>0) and  (i<=len(woe)-2):
            var.loc[((var[var_name]>cut[i])&(var[var_name]<=cut[i+1])),woe_name]=woe[i] #    中间的分箱区间   ，，数手指头就很清楚了
        else:
            var.loc[(var[var_name]>cut[len(woe)-1]),woe_name]=woe[len(woe)-1]   # 大于最后一个分箱区间的 上限值，最后一个值是正无穷
    return var
x1_name='RevolvingUtilizationOfUnsecuredLines'
x2_name='age'
x3_name='NumberOfTime30-59DaysPastDueNotWorse'
x7_name='NumberOfTimes90DaysLate'
x9_name='NumberOfTime60-89DaysPastDueNotWorse'

train=trans_woe(train,x1_name,woex1,cutx1)
train=trans_woe(train,x2_name,woex2,cutx2)
train=trans_woe(train,x3_name,woex3,cutx3)
train=trans_woe(train,x7_name,woex7,cutx7)
train=trans_woe(train,x9_name,woex9,cutx9)

Y=train['SeriousDlqin2yrs']   #因变量
#自变量，剔除对因变量影响不明显的变量
X=train.iloc[:,-5:]
print(X.head())


test=trans_woe(test,x1_name,woex1,cutx1)
test=trans_woe(test,x2_name,woex2,cutx2)
test=trans_woe(test,x3_name,woex3,cutx3)
test=trans_woe(test,x7_name,woex7,cutx7)
test=trans_woe(test,x9_name,woex9,cutx9)
#构建测试集的特征和标签
test_X=test.iloc[:,-5:]   #测试数据 特征
test_Y=test.iloc[:,0]     #测试数据  标签

import statsmodels.api as sm
X1=sm.add_constant(X)
logit=sm.Logit(Y,X1)
result=logit.fit()
print(result.summary())


#评估
from sklearn import metrics
X3=sm.add_constant(test_X)
resu = result.predict(X3)    #进行预测
fpr,tpr,threshold=metrics.roc_curve(test_Y,resu)   #评估算法
rocauc=metrics.auc(fpr,tpr)   #计算AUC
plt.figure(figsize=(8,5))  #只能在这里面设置
plt.plot(fpr,tpr,'b',label='AUC=%0.2f'% rocauc)
plt.legend(loc='lower right',fontsize=14)
plt.plot([0, 1], [0, 1], 'r--')
plt.xlim([0, 1])
plt.ylim([0, 1])
plt.xticks(fontsize=14)
plt.yticks(fontsize=14)
plt.ylabel('TPR',fontsize=16)
plt.xlabel('FPR',fontsize=16)
plt.show()


import math
coe = result.params
# 我们取600分为基础分值，PDO为20（每高20分好坏比翻一倍），好坏比取20。
p = 20/math.log(2)#比例因子
q = 600-20*math.log(20)/math.log(2)
baseScore = round(p-q*coe[0],0)
# 因为第一个是常数项
#构建评分卡时候只需要选出那些，IV值高的特征就行，最后相加得到总分

#计算分数函数
def get_score(coe,woe,factor):
    scores=[]
    for w in woe:
        score=round(coe*w*factor,0)
        scores.append(score)
    return scores

# 各项部分分数
x1 = get_score(coe[1], woex1, p)
x2 = get_score(coe[2], woex2, p)
x3 = get_score(coe[3], woex3, p)
x7 = get_score(coe[4], woex7, p)
x9 = get_score(coe[5], woex9, p)


def compute_score(series,cut,score):
    list = []
    i = 0
    while i < len(series):
        value = series[i]
        j = len(cut) - 2
        m = len(cut) - 2
        while j >= 0:
            if value >= cut[j]:
                j = -1
            else:
                j -= 1
                m -= 1
        list.append(score[m])
        i += 1
    return list
# j和m相当于j是移动光标，m是跟着j，确定数的
# list就是在x1里面挑一个值，这个值和series[i]是对应的

from pandas import Series
test['BaseScore']= (Series(np.zeros(len(test))) + baseScore)
test['x1'] = Series(compute_score(test['RevolvingUtilizationOfUnsecuredLines'], cutx1, x1))
test['x2'] = Series(compute_score(test['age'], cutx2, x2))
test['x3'] = Series(compute_score(test['NumberOfTime30-59DaysPastDueNotWorse'], cutx3, x3))
test['x7'] = Series(compute_score(test['NumberOfTimes90DaysLate'], cutx7, x7))
test['x9'] = Series(compute_score(test['NumberOfTime60-89DaysPastDueNotWorse'], cutx9, x9))
test['Score'] =baseScore - (test['x1'] + test['x2'] + test['x3'] + test['x7'] +test['x9'])
test.to_csv('F:/大数据课设/database/ScoreData0000.csv', index=False)
final = (test.loc[:,['SeriousDlqin2yrs','BaseScore', 'x1', 'x2', 'x3', 'x7', 'x9', 'Score']].head())
print(final)