数据集缺失值的处理

一. 数据集随机生成缺失值,并导出数据

def nan2(data, data1):
    '''
        随机生成缺失值的数据矩阵
        :param data: 完整数据集的数据矩阵
        :param data1: 空数据矩阵
        :return: 存在缺失值的数据矩阵
    '''
    for i in range(len(data.T)):
        num = data[:, i]
        num1 = rd.sample(range(0,len(num)), int((len(num) * 0.1)))    #10%的缺失值，保证不重复不为负值
        for j in range(len(num1)):
            num[num1[j]] = np.nan   #置空
        data1[:, i] = num
    return data1


def man():
    '''
        随机生成缺失值的数据集
        :return:
    '''
    filename = '../data/i.csv'
    filename1 = '../data/1.csv'
    Header = pd.DataFrame(pd.read_csv(filename, header=None))
    data = readcsv(filename)    #函数读取CSv数据集返回数据矩阵
    Header = np.array(Header)      #读取数据集的表头
    Header = Header[0]
    data1 = np.array(pd.DataFrame(pd.read_csv(filename)))
    data1 = nan2(data, data1)
    data1 = pd.DataFrame(data1)
    data1.to_csv(filename1, index=False, header=Header)     #数据矩阵输出为CSV文件

 运行结果：

完 整 数 据 集

Sepal.Length  Sepal.Width  Petal.Length  Petal.Width           class 0             5.1          3.5           1.4          0.2     Iris-setosa 1             4.9          3.0           1.4          0.2     Iris-setosa 2             4.7          3.2           1.3          0.2     Iris-setosa 3             4.6          3.1           1.5          0.2     Iris-setosa 4             5.0          3.6           1.4          0.2     Iris-setosa ..            ...          ...           ...          ...             ... 145           6.7          3.0           5.2          2.3  Iris-virginica 146           6.3          2.5           5.0          1.9  Iris-virginica 147           6.5          3.0           5.2          2.0  Iris-virginica 148           6.2          3.4           5.4          2.3  Iris-virginica 149           5.9          3.0           5.1          1.8  Iris-virginica

缺 失 值 数 据 集

Sepal.Length  Sepal.Width  Petal.Length  Petal.Width           class 0             5.1          NaN           1.4          0.2     Iris-setosa 1             4.9          NaN           1.4          NaN     Iris-setosa 2             4.7          3.2           1.3          0.2     Iris-setosa 3             4.6          3.1           1.5          0.2     Iris-setosa 4             5.0          3.6           1.4          NaN     Iris-setosa ..            ...          ...           ...          ...             ... 145           6.7          3.0           5.2          2.3  Iris-virginica 146           6.3          2.5           5.0          1.9  Iris-virginica 147           6.5          3.0           5.2          2.0  Iris-virginica 148           NaN          NaN           5.4          2.3  Iris-virginica 149           5.9          3.0           5.1          1.8  Iris-virginica

二. 对含有缺失值的数据集进行补充

def nanm():
    '''
        均值补充缺失值
        :return:
    '''
    filename = '../data/3.csv'
    filename1 = '../data/1.csv'
    data = pd.DataFrame(pd.read_csv(filename1))
    Header = pd.DataFrame(pd.read_csv(filename1, header=None))
    Header = np.array(Header)
    Header = Header[0]
    print(data)
    for i in range(len(data.T) - 1):
        data[Header[i]] = data[Header[i]].fillna(round(data[Header[i]].mean(), 2))#均值填充
        # data[Header[i]] = data[Header[i]].fillna(round(data[Header[i]].median(), 2))#中位数填充
        # data[Header[i]] = data[Header[i]].fillna(round(stats.mode(Header[i][0][0]), 2))#众数填充
        # data[Header[i]] = data[Header[i]].fillna(method='pad')#用前一个数据进行填充
        # data[Header[i]] = data[Header[i]].fillna(method='bfill')#用后一个数据进行填充
    data.to_csv(filename, index=False)
    print(data)

 运行代码：

缺 失 值 数 据 集	Sepal.Length  Sepal.Width  Petal.Length  Petal.Width           class 0             5.1          NaN           1.4          0.2     Iris-setosa 1             4.9          NaN           1.4          NaN     Iris-setosa 2             4.7          3.2           1.3          0.2     Iris-setosa 3             4.6          3.1           1.5          0.2     Iris-setosa 4             5.0          3.6           1.4          NaN     Iris-setosa ..            ...          ...           ...          ...             ... 145           6.7          3.0           5.2          2.3  Iris-virginica 146           6.3          2.5           5.0          1.9  Iris-virginica 147           6.5          3.0           5.2          2.0  Iris-virginica 148           NaN          NaN           5.4          2.3  Iris-virginica 149           5.9          3.0           5.1          1.8  Iris-virginica
均 值 补 充 数 据 集	Sepal.Length  Sepal.Width  Petal.Length  Petal.Width           class 0            5.10         3.06           1.4          0.2     Iris-setosa 1            4.90         3.06           1.4          1.2     Iris-setosa 2            4.70         3.20           1.3          0.2     Iris-setosa 3            4.60         3.10           1.5          0.2     Iris-setosa 4            5.00         3.60           1.4          1.2     Iris-setosa ..            ...          ...           ...          ...             ... 145          6.70         3.00           5.2          2.3  Iris-virginica 146          6.30         2.50           5.0          1.9  Iris-virginica 147          6.50         3.00           5.2          2.0  Iris-virginica 148          5.86         3.06           5.4          2.3  Iris-virginica 149          5.90         3.00           5.1          1.8  Iris-virginica
中 位 数 补 充 数 据 集	Sepal.Length  Sepal.Width  Petal.Length  Petal.Width           class 0             5.1          3.0           1.4          0.2     Iris-setosa 1             4.9          3.0           1.4          1.4     Iris-setosa 2             4.7          3.2           1.3          0.2     Iris-setosa 3             4.6          3.1           1.5          0.2     Iris-setosa 4             5.0          3.6           1.4          1.4     Iris-setosa ..            ...          ...           ...          ...             ... 145           6.7          3.0           5.2          2.3  Iris-virginica 146           6.3          2.5           5.0          1.9  Iris-virginica 147           6.5          3.0           5.2          2.0  Iris-virginica 148           5.8          3.0           5.4          2.3  Iris-virginica 149           5.9          3.0           5.1          1.8  Iris-virginica

三. 代码整合

import pandas as pd
import numpy as np
import random as rd
from scipy import stats


def readcsv(filename):
    '''
        读取csv文件
        :param filename: csv数据集的相对地址
        :return: 数据矩阵
    '''
    df = pd.read_csv(filename)  # 读文件返回DataFrame对象
    df = pd.DataFrame(df).drop(labels=['class'], axis=1)
    data_arr = np.array(df)
    return data_arr


def nan2(data, data1):
    '''
        随机生成缺失值的数据矩阵
        :param data: 完整数据集的数据矩阵
        :param data1: 空数据矩阵
        :return: 存在缺失值的数据矩阵
    '''
    for i in range(len(data.T)):
        num = data[:, i]
        num1 = rd.sample(range(0, len(num)), int((len(num) * 0.1)))  # 10%的缺失值，保证不重复不为负值
        for j in range(len(num1)):
            num[num1[j]] = np.nan  # 置空
        data1[:, i] = num
    return data1


def man():
    '''
        随机生成缺失值的数据集
        :return:
    '''
    filename = '../data/i.csv'
    filename1 = '../data/1.csv'
    Header = pd.DataFrame(pd.read_csv(filename, header=None))
    data = readcsv(filename)
    Header = np.array(Header)  # 读取数据集的表头
    Header = Header[0]
    data1 = np.array(pd.DataFrame(pd.read_csv(filename)))
    # print(pd.DataFrame(pd.read_csv(filename)))
    data1 = nan2(data, data1)
    data1 = pd.DataFrame(data1)
    data1.to_csv(filename1, index=False, header=Header)  # 数据矩阵输出为CSV文件
    # data2 = pd.read_csv(filename1)
    # print(data2)


def nanm():
    '''
        均值补充缺失值
        :return:
    '''
    filename = '../data/3.csv'
    filename1 = '../data/1.csv'
    data = pd.DataFrame(pd.read_csv(filename1))
    Header = pd.DataFrame(pd.read_csv(filename1, header=None))
    Header = np.array(Header)
    Header = Header[0]
    # print(data)
    for i in range(len(data.T) - 1):
        data[Header[i]] = data[Header[i]].fillna(round(data[Header[i]].mean(), 2))  # 均值填充
        # data[Header[i]] = data[Header[i]].fillna(round(data[Header[i]].median(), 2))# 中位数填充
        # data[Header[i]] = data[Header[i]].fillna(round(stats.mode(Header[i][0][0]), 2))# 众数填充
        # data[Header[i]] = data[Header[i]].fillna(method='pad')# 用前一个数据进行填充
        # data[Header[i]] = data[Header[i]].fillna(method='bfill')# 用后一个数据进行填充
    data.to_csv(filename, index=False)
    # print(data)


def main():
    while (True):
        print(
            "实验五菜单(先运行1再运行2保证存在缺失值的数据集)\n" + "1.对Iris数据集随机生成10%的缺失值,并将数据导入到csv中\n" + "2.对含有缺失值的Iris数据集补充,其中的缺失值用这一维的均值补充\n" + "0.退出\n")
        num = int(input("输入菜单号："))
        if num == 1:
            man()
            print("操作完成；请继续进行操作\n")
        elif num == 2:
            nanm()
            print("操作完成；请继续进行操作\n")
        else:
            print("退出系统\n")
            break


if __name__ == '__main__':
    main()

补充缺失值可参考：python实现缺失值处理_抓起的第一个娃娃的博客-CSDN博客_python缺失值处理