Python中异常值，单一值，重复值，缺失值的处理

1. 重复值，输入为DataFrame，检测是否有重复的行以及删除重复的行并生成新的DataFrame

class Duplicated():

    def __init__(self,df,subset=None,keep='first',inplace=False):

        self.subset=subset

        if isinstance(self.subset, list) and len(self.subset) > 0 and  True not in [ True for a in self.subset if a not in df.columns] :

            self.df = df[self.subset]

        elif subset == None:

            self.df = df

        else:
            raise  Exception("subset参数设置有误，正确的格式为 None 或者 '长度大于1的list，且list中的值包含在df的列中' ")

        self.keep=keep
        self.inplace=inplace

    def find_duplicated(self):
        '''
        判断哪些行是否重复，返回的是series格式
        :return:
        '''

        if self.keep not in ['first','last',False]:
            raise Exception("keep参数超出了范围")
        else:
            return self.df.duplicated(keep=self.keep)

    def drop_duplicated(self):
        '''
        删除重复的行，默认保留第一次重复出现的行，即默认keep='first'
        :return:
        '''
        if self.keep not in ['first', 'last', False]:
            raise Exception("keep参数超出了范围")
        else:
            return self.df.drop_duplicates(keep=self.keep,inplace=self.inplace)

 使用方式：

import pandas as pd
df=pd.DataFrame({"a":[1,1,1],"b":[2, 2, 3]})
dd=Duplicated(df)
print(dd.find_duplicated())
print(type(dd.find_duplicated()))
print(dd.drop_duplicated())


'''

0    False
1     True
2    False
dtype: bool
<class 'pandas.core.series.Series'>
   a  b
0  1  2
2  1  3


'''

2. 单一值，输入为DataFrame，检测是否有单一值比例过高的列以及根据阈值剔除这些列并生成新的DataFrame

class Single():


    def __init__(self,df,threshold=0.5,detect_columns=None,subset=None):
        '''
        :param df: 输入的数据为dataframe格式
        :param threshold: 如果大于该阈值，那么剔除该列
        :param detect_colunms: 检测的列
        :param subset: 对df取子集
        '''
        self.threshold = threshold
        self.subset=subset

        # 流程: df -> df[subset] -> df[subset] 判断其中的detect_columns中的单一值

        if isinstance(self.subset, list) and len(self.subset) > 0 and  True not in [ True for a in self.subset if a not in df.columns] :

            self.df = df[self.subset]

        elif subset == None:

            self.df = df

        else:
            raise  Exception("subset参数设置有误，正确的格式为 None 或者 '长度大于1的list，且list中的值包含在df的列中' ")


        if detect_columns == None:

            self.detect_columns = self.df.columns

        elif isinstance(detect_columns, list) and len(detect_columns) > 0 and True not in [ True for a in detect_columns if a not in self.df.columns] :

            self.detect_columns = detect_columns

        else:
            raise Exception("detect_columns参数填写有误，正确的格式为 None 或者 '长度大于1的list，且list中的值包含在df的列中' ")



    def get_single_columns(self):
        '''
        针对detect_columns中的列，获取大于threshold的列
        :return:
        '''
        tempdf=self.df
        result=[]
        for col in self.detect_columns:
            cntdist=tempdf[col].value_counts() # 列中每个值的分布
            ratedist=cntdist/len(tempdf[col])  # 列中每个值的比例
            if max(ratedist)>self.threshold:   #
                result.append(col)
        return result


    def drop_single(self):
        '''
        最初的df 没有发生改变
        :return:
        '''
        tempdf=self.df
        exclude_columns=self.get_single_columns()
        return  tempdf.drop(exclude_columns,axis=1) # axis=1 表示要删除列

使用方式：

df=pd.DataFrame({

    "a":[1,2,1,1,1,1,1,1,1,2,3,4],
    "b":[1,2,34,5,6,7,8,9,12,5,7,8]
})

single=Single(df,threshold=0.5)

df=single.drop_single()
print(single.get_single_columns())
print(df.shape)

'''
['a']
(12, 1)

'''

 

3.异常值，输入为DataFrame，检测某列中是否存在异常值以及删除有异常值的列并生成新的DataFrame

class Outlier():

    def __init__(self,df,method=None,detect_columns=None,subset=None):
        '''
        :param df: 输入的数据为dataframe格式
        :param threshold: 异常识别的方法
        :param detect_columns: 检测的列
        :param subset: 对df取子集
        '''
        self.method = method
        self.subset=subset

        # 流程: df -> df[subset] -> df[subset] 判断其中的detect_columns中的单一值

        if isinstance(self.subset, list) and len(self.subset) > 0 and  True not in [ True for a in self.subset if a not in df.columns] :

            self.df = df[self.subset]

        elif subset == None:

            self.df = df

        else:
            raise  Exception("subset参数设置有误，正确的格式为 None 或者 '长度大于1的list，且list中的值包含在df的列中' ")


        if detect_columns == None:

            self.detect_columns = self.df.columns

        elif isinstance(detect_columns, list) and len(detect_columns) > 0 and True not in [ True for a in detect_columns if a not in self.df.columns] :

            self.detect_columns = detect_columns

        else:
            raise Exception("detect_columns参数填写有误，正确的格式为 None 或者 '长度大于1的list，且list中的值包含在df的列中' ")


    def get_exper_bound(self,series):
        '''
        根据经验判断，[0.25Q-1.5*IQR , 0.75Q+1.5*IQR] 作为临界点
        :param series:
        :return:
        '''

        quntilelist=series.quantile([0.25,0.5,0.75])
        IQR=quntilelist[0.75]-quntilelist[0.25]
        low_bound=quntilelist[0.25]-1.5*IQR
        upper_bound=quntilelist[0.75]+1.5*IQR
        return  low_bound,upper_bound


    def get_3sigma_bound(self,series):
        '''
        [mean-3*sigma,mean+3*sigma] 作为临界点
        :param series:
        :return:
        '''
        mean=series.mean()
        std=series.std()
        low_bound=mean-3*std
        upper_bound=mean+3*std
        return low_bound,upper_bound


    def drop_outlier(self):
        '''
        剔除异常值的行,获得新的dataframe，最初的df仍然保留着
        :return:
        '''

        if self.method=="exper":
            tempdf = self.df
            for col in self.detect_columns:
                low_bound, upper_bound=self.get_exper_bound(self.df[col])
                tempdf=tempdf[tempdf[col]<=upper_bound]
                tempdf=tempdf[tempdf[col]>=low_bound]

            return tempdf


        if self.method=="3sigma":

            tempdf = self.df
            for col in self.detect_columns:
                low_bound, upper_bound=self.get_3sigma_bound(self.df[col])
                tempdf=tempdf[tempdf[col]<=upper_bound]
                tempdf=tempdf[tempdf[col]>=low_bound]

            return tempdf

    def get_outlier_details(self):
        '''
        获取每列的异常值，结果是dict形式
        :return:
        '''
        result={}
        tempdf = self.df

        if self.method=="exper":
            for col in self.detect_columns:
                series=tempdf[col]
                low_bound, upper_bound=self.get_exper_bound(self.df[col])
                upper_bound_values=series[series>upper_bound].values
                lower_bound_values=series[series<low_bound].values
                if len(lower_bound_values)>0 or len(upper_bound_values)>0:
                    result[str(col)]={"lower":lower_bound_values,"upper":upper_bound_values}
            return result

        if self.method=="3sigma":
            for col in self.detect_columns:
                series=tempdf[col]
                low_bound, upper_bound=self.get_3sigma_bound(self.df[col])
                upper_bound_values=series[series>upper_bound].values
                lower_bound_values=series[series<low_bound].values
                if len(lower_bound_values)>0 or len(upper_bound_values)>0:
                    result[str(col)]={"lower":lower_bound_values,"upper":upper_bound_values}
            return result

使用方式:

data=pd.DataFrame({
    "a":[1,2,3,4,5,6,7,7,7,7,100],
    "b":[1,2,3,4,5,6,7,7,7,7,200],
})
outlier=Outlier(data,method="exper",detect_columns=['a','b'])
res=outlier.drop_outlier()
print(res)

4.缺失值填补，输入为DataFrame，采用knn，均值，中位数，众数，指定值，前值，后值，线性插补的方式进行填充，并生成新的DataFrame

 

import pandas as pd
from fancyimpute import KNN
from sklearn.impute import SimpleImputer
import numpy as np
from ML.TOOLS import tools


def get_fill_methods():
    result = ['knn', 'mean', 'median', 'most_frequent', 'fixvalue', 'ffill', 'bfill', 'linear']
    return result


class FillMissingValue():

    def __init__(self,df,method='mode',k=3,toDF=True,missingvalue=np.NaN,fixvalue='-999',axis=0,subset=None):

        self.subset=subset

        # 流程: df -> df[subset] -> df[subset] 判断其中的detect_columns中的单一值

        if isinstance(self.subset, list) and len(self.subset) > 0 and  True not in [ True for a in self.subset if a not in df.columns] :

            self.df = df[self.subset]

        elif subset == None:

            self.df = df

        else:
            raise  Exception("subset参数设置有误，正确的格式为 None 或者 '长度大于1的list，且list中的值包含在df的列中' ")

        self.method=method
        self.k=k
        self.all_columns=self.df.columns
        self.toDF=toDF
        self.missingvalue=missingvalue
        self.fixvalue=fixvalue
        self.axis=axis



    def get_missing_dist(self):
        '''
        return df
        :return:
        '''

        missing_rate_result = pd.DataFrame()
        missing_rate_result["missing_count"] = self.df.isnull().sum()
        missing_rate_result["missing_rate"] = self.df.isnull().sum() / self.df.shape[0]
        return missing_rate_result


    def fill_missing_value(self):

        tempdf = self.df

        if self.method=='knn':
            knn = KNN(k=self.k)
            result = knn.fit_transform(tempdf)
            return tools.array_2df(result,self.all_columns)

        elif self.method in ['mean','median','most_frequent']:

            imputer = SimpleImputer(missing_values=self.missingvalue, strategy=self.method)
            result=imputer.fit_transform(tempdf)
            return tools.array_2df(result,self.all_columns)


        elif self.method=='fixvalue':
            result=tempdf.fillna(self.fixvalue,axis=self.axis)
            return result


        elif self.method in ['ffill','bfill']:    # 如果缺失值最后处于第一行，采用ffill，填补会失效，反之，处于最后一行，采用bfill，填补会失效
            result=tempdf.fillna(method=self.method,axis=self.axis)
            return result

        elif self.method == 'linear':

            result=tempdf.interpolate(method=self.method)
            return result

        else:
            raise Exception("不在缺失值填补的方法里面")

使用方式：

df=pd.DataFrame({"a":[1,2,4],"b":[2, 3, np.NaN]})

for ele in get_fill_methods():
    df1=FillMissingValue(df,method=ele).fill_missing_value()
    print(ele)
    print(df1)


'''

knn
     a         b
0  1.0  2.000000
1  2.0  3.000000
2  4.0  2.692308
mean
     a    b
0  1.0  2.0
1  2.0  3.0
2  4.0  2.5
median
     a    b
0  1.0  2.0
1  2.0  3.0
2  4.0  2.5
most_frequent
     a    b
0  1.0  2.0
1  2.0  3.0
2  4.0  2.0
fixvalue
   a     b
0  1     2
1  2     3
2  4  -999
ffill
   a    b
0  1  2.0
1  2  3.0
2  4  3.0
bfill
   a    b
0  1  2.0
1  2  3.0
2  4  NaN
linear
   a    b
0  1  2.0
1  2  3.0
2  4  3.0


'''