13 变量压缩
from sklearn. model_selection import train_test_split
import sklearn. tree as tree
import sklearn. ensemble as ensemble
import pandas as pd
import sklearn. metrics as metrics
from sklearn. model_selection import GridSearchCV
import matplotlib. pyplot as plt
13.1 贷款应用
X1 品格:指客户的名誉;
X2 能力:指客户的偿还能力;
X3 资本:指客户的财务实力和财务状况;
X4 担保:指对申请贷款项担保的覆盖程度;
X5 环境:指外部经济、政策环境对客户的影响
一、主成分分析
1、数据导入
import pandas as pd
model_data = pd. read_csv( r'.\data\Loan_aply.csv' , encoding= 'gbk' )
model_data. head( )
ID X1 X2 X3 X4 X5 0 1 76.5 81.5 76.0 75.8 71.7 1 2 70.6 73.0 67.6 68.1 78.5 2 3 90.7 87.3 91.0 81.5 80.0 3 4 77.5 73.6 70.9 69.8 74.8 4 5 85.6 68.5 70.0 62.2 76.5
data = model_data. loc[ : , 'X1' : ]
data. head( )
X1 X2 X3 X4 X5 0 76.5 81.5 76.0 75.8 71.7 1 70.6 73.0 67.6 68.1 78.5 2 90.7 87.3 91.0 81.5 80.0 3 77.5 73.6 70.9 69.8 74.8 4 85.6 68.5 70.0 62.2 76.5
2、查看相关系数矩阵,判定做变量降维的必要性(非必须)
corr_matrix = data. corr( method= 'pearson' )
corr_matrix
X1 X2 X3 X4 X5 X1 1.000000 0.726655 0.825342 0.676314 0.685563 X2 0.726655 1.000000 0.929080 0.938382 0.841413 X3 0.825342 0.929080 1.000000 0.883457 0.733482 X4 0.676314 0.938382 0.883457 1.000000 0.762563 X5 0.685563 0.841413 0.733482 0.762563 1.000000
3、做主成分之前,进行中心标准化
from sklearn import preprocessing
data = preprocessing. scale( data)
data
array([[-0.26099897, 0.63617414, 0.19936295, 0.37986418, -0.44786189],
[-0.82713645, -0.24452296, -0.69140767, -0.42813421, 0.46669092],
[ 1.10156919, 1.23712039, 1.79002477, 0.97799286, 0.66843051],
[-0.16504347, -0.1823561 , -0.34146207, -0.24974496, -0.0309334 ],
[ 0.61219612, -0.71077436, -0.43690178, -1.04724986, 0.1977048 ],
[ 0.55462281, 0.39786786, 0.65535267, 1.28230394, 0.1977048 ],
[ 1.41822235, 1.93131692, 1.41887034, 1.81747171, 2.28234725],
[ 0.51624061, -0.87655264, -0.56415473, -0.77441924, -1.16067512],
[-2.06496245, -1.55002689, -1.77305771, -1.19415866, -1.3489654 ],
[-0.88470975, -0.63824637, -0.25662677, -0.76392575, -0.82444246]])
4、使用sklearn的主成分分析,用于判断保留主成分的数量
from sklearn. decomposition import PCA
'''
说明:1、第一次的n_components参数应该设的大一点
说明:2、观察explained_variance_ratio_和explained_variance_的取值变化,
建议explained_variance_ratio_累积大于0.85,explained_variance_需要保留的
最后一个主成分大于0.8,
'''
pca= PCA( n_components= 3 )
pca. fit( data)
print ( pca. explained_variance_)
print ( pca. explained_variance_ratio_)
[4.67909448 0.42595504 0.33051612]
[0.84223701 0.07667191 0.0594929 ]
pca = PCA( n_components= 1 ) . fit( data)
newdata= pca. fit_transform( data)
citi10_pca= model_data. join( pd. DataFrame( newdata) )
citi10_pca
ID X1 X2 X3 X4 X5 0 0 1 76.5 81.5 76.0 75.8 71.7 0.267437 1 2 70.6 73.0 67.6 68.1 78.5 -0.775172 2 3 90.7 87.3 91.0 81.5 80.0 2.603684 3 4 77.5 73.6 70.9 69.8 74.8 -0.440204 4 5 85.6 68.5 70.0 62.2 76.5 -0.678219 5 6 85.0 79.2 80.3 84.4 76.5 1.389937 6 7 94.0 94.0 87.5 89.5 92.0 3.960102 7 8 84.6 66.9 68.8 64.8 66.4 -1.310850 8 9 57.7 60.4 57.4 60.8 65.0 -3.530640 9 10 70.0 69.2 71.7 64.9 68.9 -1.486076
'''通过主成分在每个变量上的权重的绝对值大小,确定每个主成分的代表性
'''
pd. DataFrame( pca. components_) . T
0 0 0.413490 1 0.472893 2 0.465599 3 0.454653 4 0.426504
由citi10_pca的‘0’标签列可知,6行按主成分打分为3.96。
3.96如何得到?
1、首先打印出出主成分计算公式右端的系数。x = pd.DataFrame(pca.components_).T 2、提取主成分的X值,选取主成分打分最高的data[6] 3、按照公式计算,data[6].dot(x.values)
主成分到底是什么?
把原始信息的方差(即信息)反映到主成分上(每个主成分两两之间是正交的)。选取前n个能反映大部分信息的主成分。就好比把一个具有一定相关性的空间点,建立一个三维坐标系,发现数据在某一维上的变化(即方差)不大(譬如几乎落在xy平面上),那么数据就可以简化到二维平面xy上。
x = pd. DataFrame( pca. components_) . T
x. values
array([[0.41348998],
[0.47289329],
[0.46559941],
[0.45465337],
[0.42650378]])
data[ 6 ]
array([1.41822235, 1.93131692, 1.41887034, 1.81747171, 2.28234725])
data[ 6 ] . dot( x. values)
array([3.96010211])
13.2 经济发展
X1 GDP
X2 人均GDP
X3 工业增加值
X4 第三产业增加值
X5 固定资产投资
X6 基本建设投资
X7 社会消费品零售总额
X8 海关出口总额
X9 地方财政收入
一、主成分分析
1、数据导入
import pandas as pd
import os
model_data = pd. read_csv( r'.\data\cities_10.csv' , encoding= 'gbk' )
model_data. head( )
AREA X1 X2 X3 X4 X5 X6 X7 X8 X9 0 辽宁 5458.2 13000 1376.2 2258.4 1315.9 529.0 2258.4 123.7 399.7 1 山东 10550.0 11643 3502.5 3851.0 2288.7 1070.7 3181.9 211.1 610.2 2 河北 6076.6 9047 1406.7 2092.6 1161.6 597.1 1968.3 45.9 302.3 3 天津 2022.6 22068 822.8 960.0 703.7 361.9 941.4 115.7 171.8 4 江苏 10636.3 14397 3536.3 3967.2 2320.0 1141.3 3215.8 384.7 643.7
data = model_data. loc[ : , 'X1' : ]
data. head( )
X1 X2 X3 X4 X5 X6 X7 X8 X9 0 5458.2 13000 1376.2 2258.4 1315.9 529.0 2258.4 123.7 399.7 1 10550.0 11643 3502.5 3851.0 2288.7 1070.7 3181.9 211.1 610.2 2 6076.6 9047 1406.7 2092.6 1161.6 597.1 1968.3 45.9 302.3 3 2022.6 22068 822.8 960.0 703.7 361.9 941.4 115.7 171.8 4 10636.3 14397 3536.3 3967.2 2320.0 1141.3 3215.8 384.7 643.7
2、查看相关系数矩阵,判定做变量降维的必要性(非必须)
corr_matrix = data. corr( method= 'pearson' )
corr_matrix
X1 X2 X3 X4 X5 X6 X7 X8 X9 X1 1.000000 -0.094292 0.966506 0.979238 0.922984 0.921680 0.941148 0.637458 0.825568 X2 -0.094292 1.000000 0.112726 0.074167 0.214052 0.093483 -0.042776 0.081195 0.273145 X3 0.966506 0.112726 1.000000 0.985373 0.963159 0.939194 0.935196 0.704714 0.898016 X4 0.979238 0.074167 0.985373 1.000000 0.972862 0.939720 0.962267 0.713890 0.913364 X5 0.922984 0.214052 0.963159 0.972862 1.000000 0.971337 0.937109 0.716722 0.934549 X6 0.921680 0.093483 0.939194 0.939720 0.971337 1.000000 0.897127 0.624294 0.848004 X7 0.941148 -0.042776 0.935196 0.962267 0.937109 0.897127 1.000000 0.836272 0.928692 X8 0.637458 0.081195 0.704714 0.713890 0.716722 0.624294 0.836272 1.000000 0.881528 X9 0.825568 0.273145 0.898016 0.913364 0.934549 0.848004 0.928692 0.881528 1.000000
3、做主成分之前,进行中心标准化
from sklearn import preprocessing
data = preprocessing. scale( data)
4、使用sklearn的主成分分析,用于判断保留主成分的数量
from sklearn. decomposition import PCA
'''
说明:1、第一次的n_components参数应该设的大一点
说明:2、观察explained_variance_ratio_和explained_variance_的取值变化,
建议explained_variance_ratio_累积大于0.85,explained_variance_需要保留的
最后一个主成分大于0.8,
'''
pca= PCA( n_components= 3 )
pca. fit( data)
print ( pca. explained_variance_)
print ( pca. explained_variance_ratio_)
[8.01129553 1.22149318 0.60792399]
[0.80112955 0.12214932 0.0607924 ]
'''通过主成分在每个变量上的权重的绝对值大小,确定每个主成分的代表性
'''
pd. DataFrame( pca. components_) . T
0 1 2 0 0.353682 -0.212192 -0.247627 1 0.040555 0.942778 -0.127315 2 0.364148 -0.009845 -0.183606 3 0.367584 -0.045377 -0.154498 4 0.365917 0.095213 -0.165382 5 0.352119 -0.023027 -0.315878 6 0.364419 -0.135241 0.150223 7 0.297565 0.048047 0.802794 8 0.355405 0.183830 0.265924
二、因子分析
因子分析的概念很多,作为刚入门的人,我们可以认为因子分析是主成分分析的延续
1、调用因子分析包
from fa_kit import FactorAnalysis
from fa_kit import plotting as fa_plotting
fa = FactorAnalysis. load_data_samples(
data,
preproc_demean= True ,
preproc_scale= True
)
fa. extract_components( )
2、设定提取主成分的方式。
默认为“broken_stick”方法,建议使用“top_n”法
fa. find_comps_to_retain( method= 'top_n' , num_keep= 2 )
array([0, 1], dtype=int64)
3、通过最大方差法进行因子旋转
pd. DataFrame( fa. comps[ "rot" ] )
fa. rotate_components( method= 'varimax' )
fa_plotting. graph_summary( fa)
4、获取因子得分
#到目前还没有与PCA中fit_transform类似的函数,因此只能手工计算因子以下是矩阵相乘的方式计算因子:因子=原始数据(n*k)权重矩阵(k num_keep)
import numpy as np
fas = pd. DataFrame( fa. comps[ "rot" ] )
data = pd. DataFrame( data)
fa_score = pd. DataFrame( np. dot( data, fas) )
fa_score
0 1 0 -1.174241 -0.364178 1 2.095775 -0.654819 2 -1.399899 -0.870629 3 -3.265185 0.698849 4 2.386557 -0.337666 5 0.163901 2.802894 6 1.209012 0.048116 7 -2.084500 -0.322173 8 5.501759 0.105138 9 -3.433179 -1.105531
三、根据因子得分进行数据分析
a= fa_score. rename( columns= { 0 : "Gross" , 1 : "Avg" } )
citi10_fa= model_data. join( a)
citi10_fa. head( )
AREA X1 X2 X3 X4 X5 X6 X7 X8 X9 Gross Avg 0 辽宁 5458.2 13000 1376.2 2258.4 1315.9 529.0 2258.4 123.7 399.7 -1.174241 -0.364178 1 山东 10550.0 11643 3502.5 3851.0 2288.7 1070.7 3181.9 211.1 610.2 2.095775 -0.654819 2 河北 6076.6 9047 1406.7 2092.6 1161.6 597.1 1968.3 45.9 302.3 -1.399899 -0.870629 3 天津 2022.6 22068 822.8 960.0 703.7 361.9 941.4 115.7 171.8 -3.265185 0.698849 4 江苏 10636.3 14397 3536.3 3967.2 2320.0 1141.3 3215.8 384.7 643.7 2.386557 -0.337666
citi10_fa. to_csv( ".\data\citi10_fa.csv" )
from pylab import mpl
mpl. rcParams[ 'font.sans-serif' ] = [ 'SimHei' ]
mpl. rcParams[ 'axes.unicode_minus' ] = False
import matplotlib. pyplot as plt
x= citi10_fa[ 'Gross' ]
y= citi10_fa[ 'Avg' ]
label= citi10_fa[ 'AREA' ]
plt. scatter( x, y)
for a, b, l in zip ( x, y, label) :
plt. text( a, b+ 0.1 , '%s.' % l, ha= 'center' , va= 'bottom' , fontsize= 14 )
plt. show( )
四、变量筛选(Var_Select函数)
def Var_Select ( orgdata, k, alphaMax= 10 , alphastep= 0.2 ) :
"""
orgdata-需要信息压缩的数据框
k-预期最大需要保留的最大变量个数,实际保留数量不能多于这个数值
alphaMax-SparsePCA算法惩罚项的最大值,一般要到5才会取得比较理想的结果
alphastep-SparsePCA算法惩罚项递增的步长
"""
if orgdata. iloc[ : , 1 ] . count( ) > 5000 :
data = orgdata. sample( 5000 )
else :
data = orgdata
from sklearn import preprocessing
import pandas as pd
import numpy as np
from sklearn. decomposition import SparsePCA
data = preprocessing. scale( data)
n_components = k
for i in np. arange( 0.1 , alphaMax, alphastep) :
pca_model = SparsePCA( n_components= n_components, alpha= i)
pca_model. fit( data)
pca = pd. DataFrame( pca_model. components_) . T
n = data. shape[ 1 ] - sum ( sum ( np. array( pca != 0 ) ) )
if n == 0 :
global best_alpha
best_alpha = i
break
pca_model = SparsePCA( n_components= n_components, alpha= best_alpha)
pca_model. fit( data)
pca = pd. DataFrame( pca_model. components_) . T
data = pd. DataFrame( data)
score = pd. DataFrame( pca_model. fit_transform( data) )
r = [ ]
R_square = [ ]
for xk in range ( data. shape[ 1 ] ) :
for paj in range ( n_components) :
r. append( abs ( np. corrcoef( data. iloc[ : , xk] , score. iloc[ : , paj] ) [ 0 , 1 ] ) )
r_max1 = max ( r)
r. remove( r_max1)
r. append( - 2 )
r_max2 = max ( r)
R_square. append( ( 1 - r_max1 ** 2 ) / ( 1 - r_max2 ** 2 ) )
R_square = abs ( pd. DataFrame( np. array( R_square) . reshape( ( data. shape[ 1 ] , n_components) ) ) )
var_list = [ ]
for i in range ( n_components) :
vmin = R_square[ i] . min ( )
var_list. append( R_square[ R_square[ i] == vmin] [ i] . index)
news_ids = [ ]
for id in var_list:
if id not in news_ids:
news_ids. append( id )
print ( news_ids)
data_vc = orgdata. iloc[ : , np. array( news_ids) . reshape( len ( news_ids) ) ]
return data_vc
model_data = pd. read_csv( r".\data\cities_10.csv" , encoding= 'gbk' )
model_data. head( )
data = model_data. loc[ : , 'X1' : ]
data. head( )
X1 X2 X3 X4 X5 X6 X7 X8 X9 0 5458.2 13000 1376.2 2258.4 1315.9 529.0 2258.4 123.7 399.7 1 10550.0 11643 3502.5 3851.0 2288.7 1070.7 3181.9 211.1 610.2 2 6076.6 9047 1406.7 2092.6 1161.6 597.1 1968.3 45.9 302.3 3 2022.6 22068 822.8 960.0 703.7 361.9 941.4 115.7 171.8 4 10636.3 14397 3536.3 3967.2 2320.0 1141.3 3215.8 384.7 643.7
Varseled_data= Var_Select( data, k= 2 )
Varseled_data. head( )
[Int64Index([3], dtype='int64'), Int64Index([1], dtype='int64')]
X4 X2 0 2258.4 13000 1 3851.0 11643 2 2092.6 9047 3 960.0 22068 4 3967.2 14397
13.3 银行交易
CNT_TBM 柜台交易次数
CNT_ATM ATM机交易次数
CNT_POS POS机交易次数
CNT_CSC 有偿服务次数
一、主成分分析
1、数据导入
import pandas as pd
model_data = pd. read_csv( r'.\data\profile_bank.csv' , encoding= 'gbk' )
model_data. head( )
data = model_data. loc[ : , 'CNT_TBM' : 'CNT_CSC' ]
data. head( )
CNT_TBM CNT_ATM CNT_POS CNT_CSC 0 34 3 3 9 1 44 17 5 18 2 122 26 32 36 3 42 3 6 1 4 20 15 2 2
2、查看相关系数矩阵,判定做变量降维的必要性(非必须)
corr_matrix = data. corr( method= 'pearson' )
corr_matrix
CNT_TBM CNT_ATM CNT_POS CNT_CSC CNT_TBM 1.000000 0.055648 0.083624 0.198835 CNT_ATM 0.055648 1.000000 0.341161 0.242106 CNT_POS 0.083624 0.341161 1.000000 0.234055 CNT_CSC 0.198835 0.242106 0.234055 1.000000
3、做主成分之前,进行中心标准化
from sklearn import preprocessing
data = preprocessing. scale( data)
pd. DataFrame( data) . head( )
0 1 2 3 0 -0.337476 -0.791125 -0.430376 0.190946 1 -0.238617 -0.119856 -0.333915 0.933000 2 0.532487 0.311675 0.968311 2.417108 3 -0.258389 -0.791125 -0.285684 -0.468657 4 -0.475880 -0.215751 -0.478607 -0.386207
4、使用sklearn的主成分分析,用于判断保留主成分的数量
from sklearn. decomposition import PCA
'''说明:1、第一次的n_components参数应该设的大一点
说明:2、观察explained_variance_ratio_和explained_variance_的取值变化,建议explained_variance_ratio_累积大于0.85,explained_variance_需要保留的最后一个主成分大于0.8,
'''
pca= PCA( n_components= 4 )
pca. fit( data)
print ( pca. explained_variance_)
print ( pca. explained_variance_ratio_)
[1.60786876 1.00252275 0.7339482 0.65570029]
[0.40196317 0.25062818 0.18348521 0.16392343]
pca= PCA( n_components= 3 ) . fit( data)
newdata= pca. fit_transform( data)
'''通过主成分在每个变量上的权重的绝对值大小,确定每个主成分的代表性
'''
pd. DataFrame( pca. components_) . T
0 1 2 0 0.303020 0.834245 0.445132 1 0.555131 -0.377566 0.135542 2 0.559520 -0.315486 0.386716 3 0.535673 0.248894 -0.796201
二、因子分析
#因子分析的概念很多,作为刚入门的人,我们可以认为因子分析是主成分分析的延续
1、建模
from fa_kit import FactorAnalysis
from fa_kit import plotting as fa_plotting
fa = FactorAnalysis. load_data_samples(
data,
preproc_demean= True ,
preproc_scale= True
)
fa. extract_components( )
2、设定提取主成分的方式。
默认为“broken_stick”方法,建议使用“top_n”法
fa. find_comps_to_retain( method= 'top_n' , num_keep= 3 )
array([0, 1, 2], dtype=int64)
3、通过最大方差法进行因子旋转
说明:可以通过第三张图观看每个因子在每个变量上的权重,权重越高,代表性越强
pd. DataFrame( fa. comps[ "rot" ] )
fa. rotate_components( method= 'varimax' )
fa_plotting. graph_summary( fa)
4、获取因子得分
#到目前还没有与PCA中fit_transform类似的函数,因此只能手工计算因子 #以下是矩阵相乘的方式计算因子:因子=原始数据(n*k)权重矩阵(k num_keep)
import numpy as np
fas = pd. DataFrame( fa. comps[ "rot" ] )
data = pd. DataFrame( data)
fa_score = pd. DataFrame( np. dot( data, fas) )
fa_score. head( )
0 1 2 0 -0.852354 -0.294938 0.143935 1 -0.333078 -0.244334 0.939343 2 0.918067 0.593787 2.349496 3 -0.741847 -0.210507 -0.521592 4 -0.499703 -0.492714 -0.367629
a= fa_score. rename( columns= { 0 : "Gross" , 1 : "Avg" } )
profile_bank_fa= model_data. join( a)
profile_bank_fa. head( )
ID CNT_TBM CNT_ATM CNT_POS CNT_CSC CNT_TOT Gross Avg 2 0 41360 34 3 3 9 49 -0.852354 -0.294938 0.143935 1 52094 44 17 5 18 84 -0.333078 -0.244334 0.939343 2 57340 122 26 32 36 216 0.918067 0.593787 2.349496 3 76885 42 3 6 1 52 -0.741847 -0.210507 -0.521592 4 89150 20 15 2 2 39 -0.499703 -0.492714 -0.367629
三、变量筛选(Var_Select函数)
import pandas as pd
model_data = pd. read_csv( r".\data\profile_bank.csv" )
data = model_data. loc[ : , 'CNT_TBM' : 'CNT_CSC' ]
data. head( )
CNT_TBM CNT_ATM CNT_POS CNT_CSC 0 34 3 3 9 1 44 17 5 18 2 122 26 32 36 3 42 3 6 1 4 20 15 2 2
Varseled_data= Var_Select( data, k= 3 , alphaMax= 10 )
Varseled_data. head( )
[Int64Index([2], dtype='int64'), Int64Index([0], dtype='int64'), Int64Index([3], dtype='int64')]
CNT_POS CNT_TBM CNT_CSC 0 3 34 9 1 5 44 18 2 32 122 36 3 6 42 1 4 2 20 2
from VarSelec import Var_Select, Var_Select_auto
import pandas as pd
model_data = pd. read_csv( r".\data\profile_bank.csv" )
data = model_data. loc[ : , 'CNT_TBM' : 'CNT_CSC' ]
Varseled_data= Var_Select( data, k= 3 )
Varseled_data. head( )
[Int64Index([2], dtype='int64'), Int64Index([0], dtype='int64'), Int64Index([3], dtype='int64')]
CNT_POS CNT_TBM CNT_CSC 0 3 34 9 1 5 44 18 2 32 122 36 3 6 42 1 4 2 20 2
Varseled_Auto_data= Var_Select_auto( data)
Varseled_Auto_data. head( )
[Int64Index([2], dtype='int64'), Int64Index([0], dtype='int64'), Int64Index([3], dtype='int64')]
CNT_POS CNT_TBM CNT_CSC 0 3 34 9 1 5 44 18 2 32 122 36 3 6 42 1 4 2 20 2
13.4 信用卡
import pandas as pd
model_data = pd. read_csv( r'./data/creditcard_exp.csv' )
model_data. head( )
data = model_data. loc[ : , 'gender' : ]
data. head( )
gender Age Income Ownrent Selfempl dist_home_val dist_avg_income age2 high_avg edu_class 0 1 40 16.03515 1 1 99.93 15.932789 1600 0.102361 3 1 1 32 15.84750 1 0 49.88 15.796316 1024 0.051184 2 2 1 36 8.40000 0 0 88.61 7.490000 1296 0.910000 1 3 1 41 11.47285 1 0 16.10 11.275632 1681 0.197218 3 4 1 28 13.40915 1 0 100.39 13.346474 784 0.062676 2
Varseled_data= Var_Select( data, k= 5 )
Varseled_data. head( )
[Int64Index([2], dtype='int64'), Int64Index([1], dtype='int64'), Int64Index([8], dtype='int64'), Int64Index([5], dtype='int64'), Int64Index([4], dtype='int64')]
Income Age high_avg dist_home_val Selfempl 0 16.03515 40 0.102361 99.93 1 1 15.84750 32 0.051184 49.88 0 2 8.40000 36 0.910000 88.61 0 3 11.47285 41 0.197218 16.10 0 4 13.40915 28 0.062676 100.39 0