数据挖掘--数据预处理(1)

这篇博客总结在数据挖掘，数据预处理阶段常用的方法和技巧，对于kaggle和天池的比赛和适用

import pandas as pd

train_pd = pd.read_csv(path+'training.csv') #读取数据
test_pd = pd.read_csv(path+'sorted_test.csv')

train_pd.info()#查看数据类型
train_pd.describe()#数值型变量的统计信息
train_pd.describe(include=['O'])#String, object类型变量的统计信息

train_pd[['PIDN', 'Depth']]#获取dataframe里的特定列

X_train = train_pd.drop(['PIDN','SOC','pH','Ca','P','Sand'],axis=1)#删除dataframe特定列

X_train['Depth'] = X_train['Depth'].map({'Topsoil':0.0, 'Subsoil':1.0}).astype(float)#把某列里的string类型转为float

train_pd.isnull().sum()#查看每一列，值为空的数据的数量 

Y_submit = pd.DataFrame(Y_pred, columns=['Ca','P','pH','SOC','Sand'])#numpy array转成dataframe并且加上对应的列名

Y_submit.insert(loc=0, column='PIDN', value=sample[['PIDN']])#dataframe 按列号插入列（第一列）

Y_submit.to_csv(path+'submission.csv', index=False)#dataframe转excel

#SVR模型的输出只能是一维的，使用MultiOutputRegressor能处理多维输出
from sklearn.svm import SVR
from sklearn.multioutput import MultiOutputRegressor

clf = SVR(gamma='scale', C=1.0, epsilon=0.2)

multi_clf = MultiOutputRegressor(clf)

#划分验证数据集
x_train, x_valid, y_train, y_valid = train_test_split(X_train, Y_train, test_size=0.33, random_state=123)

#按分组以平均值替换缺失值
processed_df['Age'] = processed_df.groupby(['Pclass','Sex','Parch','SibSp'])['Age'].transform(lambda x: x.fillna(x.mean()))

#对title列里的Lady...值全部替换成Rare
dataset['Title'] = dataset['Title'].replace(['Lady', 'Countess','Capt', 'Col',\
 	'Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona&#