Python处理缺失数据

滤除、填补，两种方式

import numpy as np

from numpy import nan as NA 
import pandas as pd 
from pandas import DataFrame 




#处理Series数据
data=pd.Series([1,2,np.nan,4])
print(data)
print([1,2,NA])
# dropna过滤掉缺失数据，对应的索引同时删除，余下索引号不变
print(data.dropna())
# drop()过滤掉()中索引对应的数据
print(data.drop(1))
#判断是否为缺失值，是则1；不是则0
print (data.isnull())
#判断是否不是缺失值，不是则1；是则0

print(data.notnull())

输出：0    1.0
1    2.0
2    NaN
3    4.0
dtype: float64
[1, 2, nan]
0    1.0
1    2.0
3    4.0
dtype: float64
0    1.0
2    NaN
3    4.0
dtype: float64
0    False
1    False
2     True
3    False
dtype: bool
0     True
1     True
2    False
3     True
dtype: bool

#处理DataFrame数据
data1=DataFrame({'a':[1,2,3,NA,NA],'b':[4,NA,5,6,NA]})
print(data1)
# dropna默认丢弃所有含有缺失值的行
print(data1.dropna())
#只丢弃全部为缺失值的行
print(data1.dropna( how='all'))
#加入一个全为缺失值的列
data1[2]=NA
print(data1)
#只丢弃全部为缺失值的列;axis为轴线，0为行，1为列

print(data1.dropna(axis=1,how='all'))

输出： a    b
0  1.0  4.0
1  2.0  NaN
2  3.0  5.0
3  NaN  6.0
4  NaN  NaN
     a    b
0  1.0  4.0
2  3.0  5.0
     a    b
0  1.0  4.0
1  2.0  NaN
2  3.0  5.0
3  NaN  6.0
     a    b   2
0  1.0  4.0 NaN
1  2.0  NaN NaN
2  3.0  5.0 NaN
3  NaN  6.0 NaN
4  NaN  NaN NaN
     a    b
0  1.0  4.0
1  2.0  NaN
2  3.0  5.0
3  NaN  6.0
4  NaN  NaN

#
#随机建立6行3列DataFrame,数值在0-1间
df=DataFrame( np.random.rand(6,3))
df1=DataFrame(np.random.rand(6,3))
print(df)
#设置一些数为缺失值
#:4,iloc默认0到3行，右开区间；loc默认0到4行，闭区间
df1.loc[1:4,1]=df1.loc[:2,2]=NA
print(df1)
df.iloc[1:4,1]=df.iloc[:2,2]=NA
print(df)
#丢弃含有少于2个非空值的行
print(df.dropna( thresh=2))

输出： 0         1         2
0  0.576558  0.145882  0.809296
1  0.001554  0.069911  0.084797
2  0.217655  0.139214  0.552833
3  0.585376  0.910459  0.835183
4  0.006047  0.500448  0.102748
5  0.163494  0.296311  0.667189
          0         1         2
0  0.729923  0.558838       NaN
1  0.010230       NaN       NaN
2  0.288920       NaN       NaN
3  0.025187       NaN  0.282088
4  0.955668       NaN  0.719648
5  0.297154  0.777692  0.838243
          0         1         2
0  0.576558  0.145882       NaN
1  0.001554       NaN       NaN
2  0.217655       NaN  0.552833
3  0.585376       NaN  0.835183
4  0.006047  0.500448  0.102748
5  0.163494  0.296311  0.667189
          0         1         2
0  0.576558  0.145882       NaN
2  0.217655       NaN  0.552833
3  0.585376       NaN  0.835183
4  0.006047  0.500448  0.102748
5  0.163494  0.296311  0.667189

#用1来填补缺失数据,并不改变本身df额值
print( df.fillna(1))
#用不同数值来填补不同列的缺失数据
print(df.fillna({1:0.5,2:-3}))
print(df)
#对现有对象就地修改
df.fillna(22,i nplace=True)
print(df)
#用此列上一个值填补缺失值

print(df1.fillna(method='ffill'))

输出：0         1         2
0  0.922985  0.427313  1.000000
1  0.274581  1.000000  1.000000
2  0.825859  1.000000  0.427558
3  0.038837  1.000000  0.571464
4  0.425408  0.820305  0.215118
5  0.491830  0.240837  0.244575
          0         1         2
0  0.922985  0.427313 -3.000000
1  0.274581  0.500000 -3.000000
2  0.825859  0.500000  0.427558
3  0.038837  0.500000  0.571464
4  0.425408  0.820305  0.215118
5  0.491830  0.240837  0.244575
          0         1         2
0  0.922985  0.427313       NaN
1  0.274581       NaN       NaN
2  0.825859       NaN  0.427558
3  0.038837       NaN  0.571464
4  0.425408  0.820305  0.215118
5  0.491830  0.240837  0.244575
          0          1          2
0  0.922985   0.427313  22.000000
1  0.274581  22.000000  22.000000
2  0.825859  22.000000   0.427558
3  0.038837  22.000000   0.571464
4  0.425408   0.820305   0.215118
5  0.491830   0.240837   0.244575
          0         1         2
0  0.499691  0.247085       NaN
1  0.421605  0.247085       NaN
2  0.332292  0.247085       NaN
3  0.771274  0.247085  0.910545
4  0.294923  0.247085  0.555058
5  0.635277  0.030381  0.376938