评分卡模型（二数据清洗)

# -*- coding: utf-8 -*-
"""
Created on Sun Sep 16 19:04:53 2018

@author: wangxihe
"""

import os
import pandas as pd
import numbers
import numpy as np
import matplotlib.pyplot as plt
#%%
os.chdir(r'E:\spyderwork\评分卡模型\一特征构建')
allData = pd.read_csv('Idx0.csv',header = 0,encoding = 'gbk')
allData.shape
#%%
os.chdir(r'E:\spyderwork\评分卡模型\二特征清洗')
describeDf=allData.describe().T

def MissingCategorial(df,col):
    missing_vals = df[col].map(lambda x: int(x!=x))
    return sum(missing_vals)*1.0/df.shape[0]

def MissingContinuous(df,col):
    missing_vals = df[col].map(lambda x: int(np.isnan(x)))
    return sum(missing_vals) * 1.0 / df.shape[0]

#%%
allFeatures = list(allData.columns)
allFeatures.remove('target')
if 'Idx' in allFeatures:
    allFeatures.remove('Idx')
allFeatures.remove('ListingInfo')

len(allFeatures)
#%%
#检查是否有常数型变量，并且检查是类别型还是数值型变量

numerical_var = []
for col in allFeatures:
    if len(set(allData[col])) == 1:
        print(' {} :此列为常数所以删除'.format(col))
        del allData[col]
        allFeatures.remove(col)
    else:
        uniq_valid_vals = [i for i in allData[col] if i == i]
       
        uniq_valid_vals = list(set(uniq_valid_vals))
        if len(uniq_valid_vals) >= 10 and isinstance(uniq_valid_vals[0], numbers.Real):
            numerical_var.append(col)

categoric