Python 数据清洗

Python 数据清洗

删除重复数据

遍历

import pandas as pd

detail = pd.read_csv('./data/meal_order_detail.csv', index_col=0)
# print(detail['dishes_name'])
#
list1 = list(detail['dishes_name'])
print(list1)
list2 = []

for i in list1:
    if i not in list2:
        list2.append(i)

print(list2)

使用set去重复

set1 = set(list1)
print(set1)

对一列数据去重

dishes_name = detail['dishes_name'].drop_duplicates()
print(dishes_name)

对多列数据去重

shapeDet = detail.drop_duplicates(subset=['order_id', 'emp_id']).shape
print(shapeDet)

检测和处理缺失值

检测

print(detail.isnull().sum()) # 输出值是空的数目
print(detail.notnull().sum()) # 输出值不是空的数目

处理

缺失值较少的时候可以用均值、众数等填充，缺失较多时直接删除

删除法

detail_new = detail.dropna(axis=1, how='any')  # any\all
print(detail.dropna(axis=1, how='any').shape)
print(detail.shape)  # 删除之后返回新的数据，没有改变原始数据

替换法

detail_new = detail.fillna(1)  # 空值填充为1
print(detail_new.isnull().sum()) # 输出值是空的数目

插值法

线性插值

import numpy as np
from scipy.interpolate import interp1d

# import scipy.interpolate.interp1d as interp1d
x = np.array([1, 2, 3, 4, 5, 8, 9, 10])
y1 = np.array([2, 8, 18, 32, 50, 128, 162, 200])
y2 = np.array([3, 5, 7, 9, 11, 17, 19, 21])
LinearInsValue1 = interp1d(x, y1, kind='linear')
LinearInsValue2 = interp1d(x, y2, kind='linear')
print(LinearInsValue1([6, 7]))
print(LinearInsValue2([6, 7]))

拉格朗日插值法

from scipy.interpolate import lagrangeLagrangeInsValue1 = lagrange(x, y1)LagrangeInsValue2 = lagrange(x, y2)print(LagrangeInsValue1([6, 7]))print(LagrangeInsValue2([6, 7]))

样条插值法

from scipy.interpolate import BSplineSplineInsValue1 = BSpline(x, y1, 3)SplineInsValue2 = BSpline(x, y2, 3)print(SplineInsValue1([6, 7]))print(SplineInsValue2([6, 7]))

检测和处理异常值

超过平均值 ± 3倍标准差，就是异常值

def outRange(Ser1):    boolInt = (Ser1.mean() - 3 * Ser1.std() > Ser1 | Ser1.mean() + 3 * Ser1.std() < Ser1)    index = np.arange(Ser1.shape[0])[boolInt]    outrange = Ser1.iloc[index]    return outrange