这一部分主要是针对样本数据做特征工程,会用到一些基本的特征工作知识,并熟悉基本的流程代码。
1、缺失值填充
#按照平均数填充数值型特征
train_data[numerical_fea] = train_data[numerical_fea].fillna(train_data[numerical_fea].median())
test_data[numerical_fea] = test_data[numerical_fea].fillna(test_data[numerical_fea].median())
#按照众数填充类别特征
train_data[category_fea] = train_data[category_fea].fillna(train_data[category_fea].mode())
test_data[category_fea] = test_data[category_fea].fillna(test_data[category_fea].mode())
2、对象性特征转换数值
train_data['employmentLength'].replace('10+ years','10 years',inplace=True)
train_data['employmentLength'].replace('< 1 year','0 years',inplace=True)
test_data['employmentLength'].replace('10+ years','10 years',inplace=True)
test_data['employmentLength'].replace('< 1 year','0 years',inplace=True)
train_data['employmentLength'] = train_data['employmentLength'].apply(lambda x: x if pd.isnull(x) else np.int8(x.split()[0]))
test_data['employmentLength'] = test_data['employmentLength'].apply(lambda x: x if pd.isnull(x) else np.int8(x.split()[0]))
train_data['earliesCreditLine'] = train_data['earliesCreditLine'].apply(lambda x: int(x[-4:]))
test_data['earliesCreditLine'] = test_data['earliesCreditLine'].apply(lambda x: int(x[-4:]))
#过滤数值型类别特征
def get_numerical_serial_fea(data,feas):
numerical_serial_fea = []
numerical_noserial_fea = []
for fea in feas:
temp = data[fea].nunique()
if temp <= 10:
numerical_noserial_fea.append(fea)
continue
numerical_serial_fea.append(fea)
return numerical_serial_fea,numerical_noserial_fea
numerical_serial_fea,numerical_noserial_fea = get_numerical_serial_fea(train_data,numerical_fea)
for data in [train_data,test_data]:
data['grade'] = data['grade'].map({'A':1,'B':2,'C':3,'D':4,'E':5,'F':6,'G':7})
3、异常值处理
def find_outliers_by_3sigm(data,fea):
data_std = np.std(data[fea])
data_mean = np.mean(data[fea])
outliers_cut_off = data_std *3
lower_rule = data_mean - outliers_cut_off
upper_rule = data_mean + outliers_cut_off
data[fea+'_outliers'] = data[fea].apply(lambda x: str('异常值') if x > upper_rule or x < lower_rule else '正常值')
return data
for fea in numerical_fea:
data1 = find_outliers_by_3sigm(train_data,fea)
print(data1[fea+'_outliers'].value_counts())
## 删除异常值
for fea in numerical_fea:
train_data = train_data[train_data[fea+'_outliers'] == '正常值']
train_data = train_data.reset_index(drop=True)