输出缺失值所占比例
def missing_data(data):
total = data.isnull().sum()
percent = (data.isnull().sum()/data.isnull().count()*100)
tt = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
types = []
for col in data.columns:
dtype = str(data[col].dtype)
types.append(dtype)
tt['Types'] = types
return(np.transpose(tt))
缺失值填充
众数:mode() | 平均数:mean() | 中位数:median
train.product_type[train.product_type.isnull()]=train.product_type.dropna().mode().values
绘图表示特征之间的相关系数
# 计算所有特征值每两个之间的相关系数,并作图表示。
corrmat = train.corr()# 得到相关系数
f,ax = plt.subplots(figsize = (12,9))
sns.heatmap(corrmat, vmax = .8, square = True)#热点图
# 取出相关性最大的前十个,做出热点图表示
k = 10 # number of variables for heatmap
cols = corrmat.nlargest(k, 'SalePrice')['SalePrice'].index
cm = np.corrcoef(train[cols].values.T)
sns.set(font_scale=1.25)
hm = sns.heatmap(cm, cbar=True, annot=True, square=True, fmt='.2f', annot_kws={
'size': 10}, yticklabels=cols.values, xticklabels=cols.values)
plt.show()
时间戳转化为日期
time_format = '%Y-%m-%d %H:%M:%S'
def get_date(timestamp) :
time_local = time.localtime(timestamp)
dt = time.strftime(time_format,time_local)
return dt
train_df['action_date'] = train_df['timestamp_of_behavior'].apply(get_date)
train_df['action_date'] = pd.to_datetime(train_df['action_date'])
train_df['action_day'] = train_df['action_date'].dt.day
train_df['action_hour'] = train_df['action_date'].dt.hour
train_df['action_minute'] = train_df['action_date'].dt.minute
train_df['action_week'] = train_df['action_date'].dt.weekday
离散属性数字化
def encode_count(df,column_name):
lbl = preprocessing.LabelEncoder()
lbl.fit(list(df[column_name].values))
df[column_name] = lbl.transform(list(df[column_name].values))
return df
train_df = encode_count(train_df,'gender&