Task2 数据分析
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import datetime
import warnings
warnings.filterwarnings('ignore')
data_train = pd.read_csv(r'...\train.csv')
data_test_a = pd.read_csv(r'...\testA.csv')
data_train.head(3).append(data_train.tail(3))
print(f"There are {data_train.isnull().any().sum()} columns in train dataset with missing values.")
have_null_fea_dict = (data_train.isnull().sum()/len(data_train)).to_dict()
fea_null_moreThanHalf = {}
for key,value in have_null_fea_dict.items():
if value > 0.5:
fea_null_moreThanHalf[key] = value
missing = data_train.isnull().sum()/len(data_train)
missing = missing[missing > 0]
missing.sort_values(inplace=True)
missing.plot.bar()
one_value_fea = [col for col in data_train.columns if data_train[col].nunique() <= 1]
one_value_fea_test = [col for col in data_test_a.columns if data_test_a[col].nunique() <= 1]
numerical_fea = list(data_train.select_dtypes(exclude=['object']).columns)
category_fea = list(filter(lambda x: x not in numerical_fea,list(data_train.columns)))
def get_numerical_serial_fea(data,feas):
numerical_serial_fea = []
numerical_noserial_fea = []
for fea in feas:
temp = data[fea].nunique()
if temp <= 10:
numerical_noserial_fea.append(fea)
continue
numerical_serial_fea.append(fea)
return numerical_serial_fea,numerical_noserial_fea
numerical_serial_fea,numerical_noserial_fea = get_numerical_serial_fea(data_train,numerical_fea)
data_train['term'].value_counts()
data_train['homeOwnership'].value_counts()
data_train['verificationStatus'].value_counts()
data_train['initialListStatus'].value_counts()
data_train['applicationType'].value_counts()
data_train['policyCode'].value_counts()
data_train['n11'].value_counts()
ata_train['n12'].value_counts()
f = pd.melt(data_train, value_vars=numerical_serial_fea)
g = sns.FacetGrid(f, col="variable", col_wrap=2, sharex=False, sharey=False)
g = g.map(sns.distplot, "value")
data_train['grade'].value_counts()
data_train['subGrade'].value_counts()
data_train['employmentLength'].value_counts()
data_train['issueDate'].value_counts()
data_train['earliesCreditLine'].value_counts()
data_train['isDefault'].value_counts()