使用卡方检验进行自动化分箱(计算卡方统计量代码)

 从结果来看，还需要人工进行调整，但是结论还是蛮有趣的。比如说按年龄分箱，会把某一个年龄作为一个区间，而这个年龄的逾期情况确实较高，人工操作是不会这么考虑的。

# -*- coding: utf-8 -*-
"""
@author: jiangdawei1
"""

import pandas as pd
import numpy as np
from scipy.stats import chi2

#导入数据
df = pd.read_csv(u'test.csv')

#计算卡方统计量
def cal_chi2(input_df, var_name, Y_name):
    '''
    df = input_df[[var_name, Y_name]]
    var_values = sorted(list(set(df[var_name])))
    Y_values = sorted(list(set(df[Y_name])))
    #用循环的方式填充
    chi2_result = pd.DataFrame(index=var_values, columns=Y_values)    
    for var_value in var_values:
        for Y_value in Y_values:
            chi2_result.loc[var_value][Y_value] = \
            df[(df[var_name]==var_value)&(df[Y_name]==Y_value)][var_name].count()
    '''
    input_df = input_df[[var_name, Y_name]]
    all_cnt = input_df[Y_name].count()
    all_0_cnt = input_df[input_df[Y_name] == 0].shape[0]
    all_1_cnt = input_df[input_df[Y_name] == 1].shape[0]
    expect_0_ratio = all_0_cnt * 1.0 / all_cnt 
    expect_1_ratio = all_1_cnt * 1.0 / all_cnt 
    #对变量的每个值计算实际个数，期望个数，卡方统计量
    var_values = sorted(list(set(input_df[var_name])))
    actual_0_cnt = []; actual_1_cnt = []; actual_all_cnt = []; 
    expect_0_cnt = []; expect_1_cnt = []; 
    chi2_value = []
    for value in var_values:
        actual_0 = input_df[(input_df[var_name]==value)&(input_df[Y_name]==0)].shape[0]
        actual_1 = input_df[(input_df[var_name]==value)&(input_df[Y_name]==1)].shape[0]
        actual_all = actual_0 + actual_1
        expect_0 = actual_all * expect_0_ratio
        expect_1 = actual_all * expect_1_ratio
        chi2_0 = (expect_0 - actual_0)**2 / expect_0
        chi2_1 = (expect_1 - actual_1)**2 / expect_1
        actual_0_cnt.append(actual_0)
        actual_1_cnt.append(actual_1)
        actual_all_cnt.append(actual_all)
        expect_0_cnt.append(expect_0)
        expect_1_cnt.append(expect_1)
        chi2_value.append(chi2_0 + chi2_1)
        
    chi2_result = pd.DataFrame({'actual_0':actual_0_cnt, 'actual_1':actual_1_cnt, 'expect_0':expect_0_cnt, \
                                'expect_1':expect_1_cnt, 'chi2_value':chi2_value, var_name+'_start':var_values, \
                                var_name+'_end':var_values}, \
                                columns=[var_name+'_start', var_name+'_end', 'actual_0', 'actual_1', 'expect_0', 'expect_1', 'chi2_value'])
    
    return chi2_result, var_name

#定义合并区间的方法
def merge_area(chi2_result, var_name, idx, merge_idx):
    #按照idx和merge_idx执行合并
    chi2_result.ix[idx, 'actual_0'] = chi2_result.ix[idx, 'actual_0'] + chi2_result.ix[merge_idx, 'actual_0']
    chi2_result.ix[idx, 'actual_1'] = chi2_result.ix[idx, 'actual_1'] + chi2_result.ix[merge_idx, 'actual_1']
    chi2_result.ix[idx, 'expect_0'] = chi2_result.ix[idx, 'expect_0'] + chi2_result.ix[merge_idx, 'expect_0']    
    chi2_result.ix[idx, 'expect_1'] = chi2_result.ix[idx, 'expect_1'] + chi2_result.ix[merge_idx, 'expect_1']   
    chi2_0 = (chi2_result.ix[idx, 'expect_0'] - chi2_result.ix[idx, 'actual_0'])**2 / chi2_result.ix[idx, 'expect_0']
    chi2_1 = (chi2_result.ix[idx, 'expect_1'] - chi2_result.ix[idx, 'actual_1'])**2 / chi2_result.ix[idx, 'expect_1']
    chi2_result.ix[idx, 'chi2_value'] = chi2_0 + chi2_1
    #调整每个区间的起始值
    if idx < merge_idx:
        chi2_result.ix[idx, var_name+'_end'] = chi2_result.ix[merge_idx, var_name+'_end']
    else:
        chi2_result.ix[idx, var_name+'_start'] = chi2_result.ix[merge_idx, var_name+'_start']
    chi2_result = chi2_result.drop([merge_idx])
    chi2_result = chi2_result.reset_index(drop=True)
    
    return chi2_result

#自动进行分箱，使用最大区间限制
def chiMerge_maxInterval(chi2_result, var_name, max_interval=5):
    groups = chi2_result.shape[0]
    while groups > max_interval:
        min_idx = chi2_result[chi2_result['chi2_value']==chi2_result['chi2_value'].min()].index.tolist()[0]
        if min_idx == 0:
            chi2_result = merge_area(chi2_result, var_name, min_idx, min_idx+1)
        elif min_idx == groups-1:    
            chi2_result = merge_area(chi2_result, var_name, min_idx, min_idx-1)
        else:
            if chi2_result.loc[min_idx-1, 'chi2_value'] > chi2_result.loc[min_idx+1, 'chi2_value']:
                chi2_result = merge_area(chi2_result, var_name, min_idx, min_idx+1)
            else:
                chi2_result = merge_area(chi2_result, var_name, min_idx, min_idx-1)
        groups = chi2_result.shape[0]

    return chi2_result