特征工程之数据预处理中的缺失值处理（使用pandas和sklearn)

原则：
1.缺失数据的比例
2.数据的重要性
方法：
如果缺失的数据不重要或者缺失的很多可以直接删除。
删除：
1.删除所有包含缺失数据的行

data=data.dropna(axis=0)

2.删除所有包含缺失数据的列

data=data.dropna(axis=1)

3.删除缺失的某个数据列如：删除a,b列

data = data.drop(['a', 'b'], axis=1)

如果缺失数据重要或者缺失的不多进行填充。
填充：
1.用均值、众数、中位数等填充

#固定值填充
data_all['Age'] = data_all['Age'].fillna(20)
#均值填充
data_all['Age'] = data_all['Age'].fillna(data_all['Age'].mean())
#众数填充
data_all['Age'] = data_all['Age'].fillna(data_all['Age'].mode())
#用上下数据进行填充
data_all['Age'] = data_all['Age'].fillna(method='pad')
data_all['Age'] = data_all['Age'].fillna(method='bfill')
#用插值法填充
data_all['Age'] = data_all['Age'].interpolate()
#缺失的为0，不缺失的为1填充。
ad_info['spreadAppId'] = ad_info['spreadAppId'].fillna(0)
ad_info['Has_spreadAppId'] = ad_info['spreadAppId'].apply(lambda x: 0 if x ==0 else 1)

2.用没有缺失的数据当训练集，缺失的数据当测试集，用机器学习的方法去预测填充。

import pandas as pd
ad_info=pd.read_csv("G:/ctr/ad_info.zip",names=['adId','billId','primId','creativeType','intertype','spreadAppId'])
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import GridSearchCV
from xgboost import XGBClassifier
from sklearn.model_selection import train_test_split
# 数据相关信息查看
# print("前10行数据：\n", ad_info.head(10))
# print("数据的统计分析：\n", ad_info.describe())
# print("数据的情况：\n", ad_info.info())
# print("数据的列：\n", ad_info.keys())
# print("数据的大小：\n", ad_info.shape)
#数据数值化
# is_not_number=['billId']
# for i in is_not_number:
#     print("属性为：", i)
#     print(ad_info[i].describe())
#     print(ad_info[i].unique())
#     print(ad_info[i].head())
le=LabelEncoder()
ad_info['billId']=le.fit_transform(ad_info['billId'])
# print(ad_info.info())
#缺失数据处理
ad_info['spreadAppId'] = ad_info['spreadAppId'].fillna(0)
ad_info['Has_spreadAppId'] = ad_info['spreadAppId'].apply(lambda x: 0 if x ==0 else 1)
#特征挖掘
# print(ad_info.keys())
all_feature=['adId', 'billId', 'primId', 'creativeType', 'intertype', 'spreadAppId']
# for i in all_feature:
#     print("属性为：", i)
#     print(ad_info[i].describe())
#     print(ad_info[i].unique())
#     print(len(ad_info[i].unique()))
#     print(ad_info[i].head())
cols=['billId', 'primId', 'creativeType', 'intertype']
train_X=ad_info[ad_info['spreadAppId']!=0][cols]
train_Y=ad_info[ad_info['spreadAppId']!=0]['spreadAppId']
test_X=ad_info[ad_info['spreadAppId']==0][cols]
#
# 数据分割
X_train, X_val, Y_train, Y_val = train_test_split(train_X, train_Y, test_size=0.3, random_state=40)
# 模型选择
# GradientBoosting
# model =KNeighborsClassifier( weights='uniform',algorithm='auto',
# leaf_size=30, p=2, metric='minkowski',metric_params=None, n_jobs=1)
# gb_param_grid = {'n_neighbors': [2,3,4,5,6]}
# gb_grid = GridSearchCV(model, gb_param_grid, cv=10, verbose=1)
# gb_grid.fit(train_X, train_Y)
# print('最佳参数组合' + str(gb_grid.best_params_))
# print('最佳学习器的性能评分' + str(gb_grid.best_score_))
model=XGBClassifier()
model.fit(X_train, Y_train)
accuracy1 = model.score(X_train, Y_train)
print("训练数据的准确率：", accuracy1)
accuracy2 = model.score(X_val, Y_val)
print("验证数据的准确率: ", accuracy2)
#预测
test_X['spreadAppId']=model.predict(test_X)
ad_info.loc[(ad_info['spreadAppId']==0), 'spreadAppId'] =test_X['spreadAppId']
print(ad_info.info())
print(ad_info.head(50))
print(ad_info.keys())

数据集使用的是华为算法精英赛赛的ad_info。。。