一、数据预处理
import pandas as pd
import numpy as np
from matplotlib import pyplot as plt
import seaborn as sns
1 导入数据
trainfile = '../input/titanic/train.csv'
testfile = '../input/titanic/test.csv'
train = pd.read_csv(trainfile)
test = pd.read_csv(testfile)
data_all = pd.concat([train,test]) # 连接训练集和测试集,统一处理
data_all.info()
data_all.isna().sum()
2 数据探索
2.1 定类、定序数据分析
f, [ax1, ax2, ax3] = plt.subplots(1, 3, figsize=(20, 4))
sns.countplot(x='Pclass', hue='Survived', data=data_all, ax=ax1)
sns.countplot(x='Sex', hue='Survived', data=data_all, ax=ax2)
sns.countplot(x='Embarked', hue='Survived', data=data_all, ax=ax3)
ax1.set_title('Feature analysis-Pclass ')
ax2.set_title('Feature analysis-Sex')
ax3.set_title('Feature analysis-Embarked')
f.suptitle('Analysis of classified/ordered data', size=20, y=1.1) # 指定子图的标题
2.2 定距数据分析
f, [ax1, ax2, ax3, ax4] = plt.subplots(1, 4, figsize=(20, 4))
sns.countplot(x='SibSp', hue='Survived', data=data_all, ax=ax1)
sns.countplot(x='Parch', hue='Survived', data=data_all, ax=ax2)
sns.violinplot(x='Survived',y='Fare',data=data_all, ax=ax3)
sns.violinplot(x='Survived',y='Age',data=data_all, ax=ax4)
ax1.set_title('Feature analysis-SibSp')
ax2.set_title('Feature analysis-Parch')
ax3.set_title('Feature analysis-Fare')
ax4.set_title('Feature analysis-Age')
f.suptitle('Analysis of distance data', size=20, y=1.1) # 指定子图的标题
3 数据清洗
data_all.info()
data_all.isna().sum()
3.1 Name、Age的处理
data_all['Title']=data_all['Name'].apply(lambda x: x.split(',')[1].split('.')[0].strip()) # 取名字中的Title。ps:strip()去除前后的空格
data_all.Title.value_counts()
pd.crosstab(data_all.Title,data_all.Sex) # Title 和 Sex的交叉表(结果显示只有“Dr”,“title”都属于2种性别)
data_all[(data_all.Title=='Dr')&(data_all.Sex=='female')] # 找到“Dr”为“female”这条数据(结果显示female“ Dr”的PassengerId为“ 797”)
# 映射其他“Title”
m={'Capt':'Rareman',
'Col':'Rareman',
'Don':'Rareman',
'Dona':'Rarewoman',
'Dr':'Rareman',
'Jonkheer':'Rareman',
'Lady':'Rarewoman',
'Major':'Rareman',
'Master':'Master',
'Miss':'Miss',
'Mlle':'Rarewoman',
'Mme':'Rarewoman',
'Mr':'Mr',
'Mrs':'Mrs',
'Ms':'Rarewoman',
'Rev':'Mr',
'Sir':'Rareman',
'the Countess':'Rarewoman'
}
data_all.Title=data_all.Title.map(m)
data_all.loc[data_all.PassengerId==797,'Title']='Rarewoman' # 将 female 'Dr' 的Title映射为 'Rarewoman'
data_all.Title.value_counts()
print(data_all[data_all.Title=='Mr']['Age'].describe())
print(data_all[data_all.Title=='Miss']['Age'].describe()) # 可以看出其中包含女孩
print(data_all[data_all.Title=='Mrs']['Age'].describe())
print(data_all[data_all.Title=='Master']['Age'].describe()) # 可以看出主要是男孩
print(data_all[data_all.Title=='Rareman']['Age'].describe())
print(data_all[data_all.Title=='Rarewoman']['Age'].describe())
# 孩子往往有较高的成活率,构造属性Girl
data_all.Age.fillna(999,inplace=True)
def girl(data_all):
# 1)对于具有年龄记录的“Miss”,我们可以简单地确定“Miss”是否是按年龄划分的小女孩
if (data_all.Age!=999)&(data_all.Title=='Miss')&(data_all.Age<=14):
return 'Girl'
# 2)对于没有年龄记录的“Miss”,我们使用(Parch!= 0),因为如果是小女孩,她很可能会由父母陪伴
elif (data_all.Age==999)&(data_all.Title=='Miss')&(data_all.Parch!=0):
return 'Girl'
else:
return data_all.Title
data_all['Title']=data_all.apply(girl,axis=1)
Tit=['Mr','Miss','Mrs','Master','Girl','Rareman','Rarewoman']
for i in Tit:
data_all.loc[(data_all.Age==999)&(data_all.Title==i),'Age']=data_all.loc[data_all.Title==i,'Age'].median()
3.2 Embarked的处理
# 因为Embarked的缺失值较少,所以采用众数进行填充
data_all.groupby('Embarked')['Embarked'].count() # 对Embarked列分类计数,结果显示众数是"S"
data_all['Embarked'].fillna('S',inplace=True) # 用"S"填充
3.3 Fare的处理
data_all[data_all.Fare.isnull()] # 结果显示缺失值来自Pclass = 3的乘客
data_all.Fare.fillna(data_all[data_all.Pclass==3]['Fare'].median(),inplace=True) # 用Pclass = 3的中位数票价来填补缺失的值
# 再次查看数据形态
data_all.info()
data_all.isna().sum()
4 属性规约
# 删除属性:PassengerId(没什么关系)、Ticket、Name、Cabin(缺失值太多)
data_cleaned = data_all.drop(columns=['PassengerId','Ticket','Name','Ticket','Cabin'])
data_cleaned.to_csv('data_cleaned.csv',index=False)
5 重新分离数据集
data_train = data_cleaned[:891]
x_train = data_train.drop(columns=['Survived'])
y_train = data_train.Survived
x_test = data_cleaned[891:].drop(columns=['Survived'])
6 数据变换
6.1 one-hot编码
x_train_s = pd.get_dummies(x_train)
x_test_s = pd.get_dummies(x_test)
6.2 对Age、Fare进行标准化
x_train_s.Age = (x_train_s.Age - x_train_s.Age.mean()) / x_train_s.Age.std()
x_train_s.Fare = (x_train_s.Fare - x_train_s.Fare.mean()) / x_train_s.Fare.std()
x_test_s.Age = (x_test_s.Age - x_test_s.Age.mean()) / x_test_s.Age.std()
x_test_s.Fare = (x_test_s.Fare - x_test_s.Fare.mean()) / x_test_s.Fare.std()