从结果来看,还需要人工进行调整,但是结论还是蛮有趣的。比如说按年龄分箱,会把某一个年龄作为一个区间,而这个年龄的逾期情况确实较高,人工操作是不会这么考虑的。
# -*- coding: utf-8 -*-
"""
@author: jiangdawei1
"""
import pandas as pd
import numpy as np
from scipy.stats import chi2
#导入数据
df = pd.read_csv(u'test.csv')
#计算卡方统计量
def cal_chi2(input_df, var_name, Y_name):
'''
df = input_df[[var_name, Y_name]]
var_values = sorted(list(set(df[var_name])))
Y_values = sorted(list(set(df[Y_name])))
#用循环的方式填充
chi2_result = pd.DataFrame(index=var_values, columns=Y_values)
for var_value in var_values:
for Y_value in Y_values:
chi2_result.loc[var_value][Y_value] = \
df[(df[var_name]==var_value)&(df[Y_name]==Y_value)][var_name].count()
'''
input_df = input_df[[var_name, Y_name]]
all_cnt = input_df[Y_name].count()
all_0_cnt = input_df[input_df[Y_name] == 0].shape[0]
all_1_cnt = input_df[input_df[Y_name] == 1].shape[0]
expect_0_ratio = all_0_cnt * 1.0 / all_cnt
expect_1_ratio = all_1_cnt * 1.0 / all_cnt
#对变量的每个值计算实际个数,期望个数,卡方统计量
var_values = sorted(list(set(input_df[var_name])))
actual_0_cnt = []; actual_1_cnt = []; actual_all_cnt = [];
expect_0_cnt = []; expect_1_cnt = [];
chi2_value = []
for value in var_values:
actual_0 = input_df[(input_df[var_name]==value)&(input_df[Y_name]==0)].shape[0]
actual_1 = input_df[(input_df[var_name]==value)&(input_df[Y_name]==1)].shape[0]
actual_all = actual_0 + actual_1
expect_0 = actual_all * expect_0_ratio
expect_1 = actual_all * expect_1_ratio
chi2_0 = (expect_0 - actual_0)**2 / expect_0
chi2_1 = (expect_1 - actual_1)**2 / expect_1
actual_0_cnt.append(actual_0)
actual_1_cnt.append(actual_1)
actual_all_cnt.append(actual_all)
expect_0_cnt.append(expect_0)
expect_1_cnt.append(expect_1)
chi2_value.append(chi2_0 + chi2_1)
chi2_result = pd.DataFrame({'actual_0':actual_0_cnt, 'actual_1':actual_1_cnt, 'expect_0':expect_0_cnt, \
'expect_1':expect_1_cnt, 'chi2_value':chi2_value, var_name+'_start':var_values, \
var_name+'_end':var_values}, \
columns=[var_name+'_start', var_name+'_end', 'actual_0', 'actual_1', 'expect_0', 'expect_1', 'chi2_value'])
return chi2_result, var_name
#定义合并区间的方法
def merge_area(chi2_result, var_name, idx, merge_idx):
#按照idx和merge_idx执行合并
chi2_result.ix[idx, 'actual_0'] = chi2_result.ix[idx, 'actual_0'] + chi2_result.ix[merge_idx, 'actual_0']
chi2_result.ix[idx, 'actual_1'] = chi2_result.ix[idx, 'actual_1'] + chi2_result.ix[merge_idx, 'actual_1']
chi2_result.ix[idx, 'expect_0'] = chi2_result.ix[idx, 'expect_0'] + chi2_result.ix[merge_idx, 'expect_0']
chi2_result.ix[idx, 'expect_1'] = chi2_result.ix[idx, 'expect_1'] + chi2_result.ix[merge_idx, 'expect_1']
chi2_0 = (chi2_result.ix[idx, 'expect_0'] - chi2_result.ix[idx, 'actual_0'])**2 / chi2_result.ix[idx, 'expect_0']
chi2_1 = (chi2_result.ix[idx, 'expect_1'] - chi2_result.ix[idx, 'actual_1'])**2 / chi2_result.ix[idx, 'expect_1']
chi2_result.ix[idx, 'chi2_value'] = chi2_0 + chi2_1
#调整每个区间的起始值
if idx < merge_idx:
chi2_result.ix[idx, var_name+'_end'] = chi2_result.ix[merge_idx, var_name+'_end']
else:
chi2_result.ix[idx, var_name+'_start'] = chi2_result.ix[merge_idx, var_name+'_start']
chi2_result = chi2_result.drop([merge_idx])
chi2_result = chi2_result.reset_index(drop=True)
return chi2_result
#自动进行分箱,使用最大区间限制
def chiMerge_maxInterval(chi2_result, var_name, max_interval=5):
groups = chi2_result.shape[0]
while groups > max_interval:
min_idx = chi2_result[chi2_result['chi2_value']==chi2_result['chi2_value'].min()].index.tolist()[0]
if min_idx == 0:
chi2_result = merge_area(chi2_result, var_name, min_idx, min_idx+1)
elif min_idx == groups-1:
chi2_result = merge_area(chi2_result, var_name, min_idx, min_idx-1)
else:
if chi2_result.loc[min_idx-1, 'chi2_value'] > chi2_result.loc[min_idx+1, 'chi2_value']:
chi2_result = merge_area(chi2_result, var_name, min_idx, min_idx+1)
else:
chi2_result = merge_area(chi2_result, var_name, min_idx, min_idx-1)
groups = chi2_result.shape[0]
return chi2_result