载入各种数据科学与可视化库
#导入warnings包,利用过滤器来实现忽略警告语句。
import warnings
warnings.filterwarnings('ignore')
import missingno as msno
import pandas as pd
from pandas import DataFrame
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np
载入训练集和测试集
#导入训练集train.csv
import pandas as pd
from pandas import DataFrame, Series
import matplotlib.pyplot as plt
Train_data = pd.read_csv('train.csv')
#导入测试集testA.csv
Test_data = pd.read_csv('testA.csv')
观察数据
#观察train首尾数据
Train_data.head().append(Train_data.tail())
id heartbeat_signals label 0 0 0.9912297987616655,0.9435330436439665,0.764677... 0.0 1 1 0.9714822034884503,0.9289687459588268,0.572932... 0.0 2 2 1.0,0.9591487564065292,0.7013782792997189,0.23... 2.0 3 3 0.9757952826275774,0.9340884687738161,0.659636... 0.0 4 4 0.0,0.055816398940721094,0.26129357194994196,0... 2.0 99995 99995 1.0,0.677705342021188,0.22239242747868546,0.25... 0.0 99996 99996 0.9268571578157265,0.9063471198026871,0.636993... 2.0 99997 99997 0.9258351628306013,0.5873839035878395,0.633226... 3.0 99998 99998 1.0,0.9947621698382489,0.8297017704865509,0.45... 2.0 99999 99999 0.9259994004527861,0.916476635326053,0.4042900... 0.0
#观察train数据集的行列信息
Train_data.shape
(100000, 3)
#观察Test_data首尾数据
Test_data.head().append(Test_data.tail())
id heartbeat_signals 0 100000 0.9915713654170097,1.0,0.6318163407681274,0.13... 1 100001 0.6075533139615096,0.5417083883163654,0.340694... 2 100002 0.9752726292239277,0.6710965234906665,0.686758... 3 100003 0.9956348033996116,0.9170249621481004,0.521096... 4 100004 1.0,0.8879490481178918,0.745564725322326,0.531... 19995 119995 1.0,0.8330283177934747,0.6340472606311671,0.63... 19996 119996 1.0,0.8259705825857048,0.4521053488322387,0.08... 19997 119997 0.951744840752379,0.9162611283848351,0.6675251... 19998 119998 0.9276692903808186,0.6771898159607004,0.242906... 19999 119999 0.6653212231837624,0.527064114047737,0.5166625...
#观察Test_data数据集的行列信息
Test_data.shape
(20000, 2)
#总览数据概况:相关统计量
Train_data.describe()
id label count 100000.000000 100000.000000 mean 49999.500000 0.856960 std 28867.657797 1.217084 min 0.000000 0.000000 25% 24999.750000 0.000000 50% 49999.500000 0.000000 75% 74999.250000 2.000000 max 99999.000000 3.000000
#获取train数据类型
Train_data.info()
<class 'pandas.core.frame.DataFrame'>
RangeIndex: 100000 entries, 0 to 99999
Data columns (total 3 columns):
# Column Non-Null Count Dtype
--- ------ -------------- -----
0 id 100000 non-null int64
1 heartbeat_signals 100000 non-null object
2 label 100000 non-null float64
dtypes: float64(1), int64(1), object(1)
memory usage: 2.3+ MB
Train_data.info
<bound method DataFrame.info of id heartbeat_signals label
0 0 0.9912297987616655,0.9435330436439665,0.764677... 0.0
1 1 0.9714822034884503,0.9289687459588268,0.572932... 0.0
2 2 1.0,0.9591487564065292,0.7013782792997189,0.23... 2.0
3 3 0.9757952826275774,0.9340884687738161,0.659636... 0.0
4 4 0.0,0.055816398940721094,0.26129357194994196,0... 2.0
... ... ... ...
99995 99995 1.0,0.677705342021188,0.22239242747868546,0.25... 0.0
99996 99996 0.9268571578157265,0.9063471198026871,0.636993... 2.0
99997 99997 0.9258351628306013,0.5873839035878395,0.633226... 3.0
99998 99998 1.0,0.9947621698382489,0.8297017704865509,0.45... 2.0
99999 99999 0.9259994004527861,0.916476635326053,0.4042900... 0.0
[100000 rows x 3 columns]>
#判断数据缺失和异常
Train_data.isnull().sum()
id 0
heartbeat_signals 0
label 0
dtype: int64
#了解预测值的分布
Train_data.label
0 0.0
1 0.0
2 2.0
3 0.0
4 2.0
...
99995 0.0
99996 2.0
99997 3.0
99998 2.0
99999 0.0
Name: label, Length: 100000, dtype: float64
Train_data.label.value_counts()
0.0 64327
3.0 17912
2.0 14199
1.0 3562
Name: label, dtype: int64
## 1) 总体分布概况(无界约翰逊分布等)
import scipy.stats as st
y = Train_data['label']
plt.figure(1); plt.title('Default')
sns.distplot(y, rug=True, bins=20)
# seaborn.distplot() 直方图,质量估计图,核密度估计图
#hist=True:表示要绘制直方图(默认情况为True),若为False,则不绘制
#kde=True:表示要绘制核密度估计图(默认情况为True),若为False,则绘制
# seaborn.distplot(a, bins=None, hist=True,
# kde=True, rug=False, fit=None,
# hist_kws=None, kde_kws=None, rug_kws=None,
# fit_kws=None, color=None, vertical=False,
# norm_hist=False, axlabel=None,
# label=None, ax=None)
# 参数解读:
# a: Series, 一维数组或列表,要输入的数据,如果设置name属性,则该名称将用于标记数据轴;
# 以下是可选参数:
# bins: matplotlib hist()的参数 或者 None,作用:指定直方图规格,若为None,则使用Freedman-Diaconis规则,该规则对数据中的离群值不太敏感,可能更适用于重尾分布的数据。
# 它使用 bin 大小
# [2∗IQR(X(:))∗numel(X)(−1/4),2∗IQR(Y(:))∗numel(Y)(−1/4)][2∗IQR(X(:))∗numel(X)(−1/4),2∗IQR(Y(:))∗numel(Y)(−1/4)] ,其中 IQR 为四分位差。
# hist:bool
# 是否绘制(标准化)直方图
# kde:bool
# 是否绘制高斯核密度估计图
# rug:bool
# 是否在支撑轴上绘制rugplot()图
# {hist,kde,rug,fit} _kws:字典
# 底层绘图函数的关键字参数
# color:matplotlib color
# 该颜色可以绘制除了拟合曲线之外的所有内容
# vertical:bool
# 如果为True,则观察值在y轴上,即水平横向的显示
plt.figure(2); plt.title('Normal')
sns.distplot(y, kde=False, fit=st.norm)
plt.figure(3); plt.title('Log Normal')
sns.distplot(y, kde=False, fit=st.lognorm)
<AxesSubplot:title={'center':'Log Normal'}, xlabel='label'>
# 2)查看skewness and kurtosis
sns.distplot(Train_data['label']);
print("Skewness: %f" % Train_data['label'].skew())
print("Kurtosis: %f" % Train_data['label'].kurt())
Skewness: 0.871005
Kurtosis: -1.009573
Train_data.skew(), Train_data.kurt()
(id 0.000000
label 0.871005
dtype: float64,
id -1.200000
label -1.009573
dtype: float64)
sns.distplot(Train_data.kurt(),color='orange',axlabel ='Kurtness')
<AxesSubplot:xlabel='Kurtness', ylabel='Density'>
print(Train_data.label.value_counts())
## 3) 查看预测值的具体频数
plt.hist(Train_data['label'], orientation = 'vertical',histtype = 'bar', color ='red')
plt.show()
0.0 64327
3.0 17912
2.0 14199
1.0 3562
Name: label, dtype: int64
用pandas_profiling生成数据报告
import pandas_profiling
pfr = pandas_profiling.ProfileReport(Train_data)
pfr.to_file("example.html")
HBox(children=(HTML(value='Summarize dataset'), FloatProgress(value=0.0, max=16.0), HTML(value='')))
HBox(children=(HTML(value='Generate report structure'), FloatProgress(value=0.0, max=1.0), HTML(value='')))
HBox(children=(HTML(value='Render HTML'), FloatProgress(value=0.0, max=1.0), HTML(value='')))
HBox(children=(HTML(value='Export report to file'), FloatProgress(value=0.0, max=1.0), HTML(value='')))