Pandas 对文本进行预处理汇总

#下载数据集
import urllib.request
import os

url = "http://biostat.mc.vanderbilt.edu/wiki/pub/Main/DataSets/titanic3.xls"
filepath = "data/tianic3.xls"
if not os.path.isfile(filepath):
    result = urllib.request.urlretrieve(url,filepath)
    print("downloaded:",result)

#使用Pandas DataFrame 读取数据
import numpy
import pandas as pd

all_df = pd.read_excel(filepath)

#把需要的字段选取到DataFrame中
cols = ['survived','name','pclass','sex'...]
all_df = all_df[cols]

#使用Pandas DataFrame 进行数据预处理

#将name字段删除
df = all_df.drop(['name'],axis=1)

#找出含有Null值的字段
all_df.isnull().sum()

#将age字段为null的数据替换为平均值
age_mean = df['age'].mean()
df['age']  = df['age'].fillna(age_mean)

#将fare字段为null的数据替换为平均值
fare_mean = df['fare'].mean()
df['fare'] = df['fare'].fillna(fare_mean)

#转换性别字段为0和1
df['sex'] = df['sex'].map({'female':0,'male':1}).astype(int)

#将embarked字段进行一位有效编码转换
x_OneHot_df = pd.get_dummies(data=df,columns=['embarked'])

#将DataFrame转换为Array
ndarray = x_OneHot_df_df.values

#将ndarray特征字段进行标准化
from sklearn import preprocessing
minmax_scale = preprocessing.MinMaxScaler(feature_range=(0,1))
scaledFeatures = minmaxscale.fit_transform(Features)

#将数据分为训练集和测试集
msk = numpy.random.rand(len(all_df)) <0.8
train_df = all_df[msk]
test_df = all_df[~msk]

print(np.random.rand(2,4))
#生成一个2行4列的0到1之间的数组
#[[0.16965512 0.97445517 0.51992353 0.73377611]
#[0.91446815 0.65995296 0.67720307 0.34809015]]

print(np.random.randn(2,4))
#具有标准正态分布
#[[-3.76215048e-04 8.66687229e-01 -2.38559669e-01 1.75060171e+00]
#[ 1.57466855e+00 8.17036401e-01 -1.05382851e+00 -1.72285071e+00]]