数据预处理11条_data = numpy.loadtxt()x = data[]-CSDN博客

本文链接：https://blog.csdn.net/qq_27015119/article/details/81011898

import pandas as pd
import numpy as np
from sklearn.preprocessing import Imputer #数据预处理的函数

#缺失值处理
np.random.seed(8)

df = pd.DataFrame(np.random.randn(6,4),columns = ['col1','col2','col3','col4'])
df.iloc[1:2,1] = np.nan
df.iloc[4,3] = np.nan
#print(df.isnull()) #查看是否有缺失值，是缺失值返回True不是则返回False
nan_row = df.isnull().any()#返回有缺失值的列
df.isnull().all() #返回所有的列
df2 = df.dropna() #去除所有的缺失值的行


nan_mean = Imputer(missing_values = 'NaN',strategy = 'mean',axis = 0) #将缺失值用列的均值填充
nan_result = nan_mean.fit_transform(df)

nan_back = df.fillna(method = 'backfill',limit = 1) #用后面的值填充缺失值 限制的次数是一次
nan_pad = df.fillna(method = 'pad') #用前面的数填充缺失值
nan0 = df.fillna(0) #用0填充缺失值

#异常值处理
df =pd.DataFrame({'col1':[1,120,3,5,2,12,13],
                  'col2':[12,17,31,53,22,32,43]})
cols = df.columns
df_zscore = df.copy()
for col in cols:
    df_col = df[col] #每列的值
    z_score = (df_col - df_col.mean())/df_col.std() #标准化处理超过预值的为异常值
    df_zscore[col] = z_score.abs() > 2.0
print(df_zscore)

#重复值处理
data1 = ['a',3]
data2 = ['b',2]
data3 = ['a',3]
data4 = ['c',2]

data = pd.DataFrame([data1,data2,data3,data4],columns = ['col1','col2'])
isduplicated = data.duplicated() #判断是否重复 是返回True否返回False

new_data1 = data.drop_duplicates() #删除重复的值
new_data2 = data.drop_duplicates(['col1']) #删除col1列重复的
new_data3 = data.drop_duplicates(['col1','col2']) #删除这两列重复的值
print(new_data1)

from sklearn.preprocessing import OneHotEncoder #数据预处理的函数

data = pd.DataFrame({'id':[3566841,6541227,3512441],'sex':['male','Female','Female'],'level':['high','low','middel']})
data_new = data.copy()

for col_num,col_name in enumerate(data): #和data.columns 一样 获取每个类的序号与列名
    col_data = data[col_name]
    col_data_type = col_data.dtype
    if col_data_type == 'object': #非数值型的
        data_new = data_new.drop(col_name,axis = 1) #删除需要转换的列
        value_sets = col_data.unique() #出现的不同的值
        for value_unique in value_sets:
            col_new_name = col_name+'_' + value_unique
            col_temp = data.iloc[:,col_num]
            new_col = (col_temp == value_unique)
            data_new[col_new_name] = new_col
print(data_new)

df2 = pd.DataFrame({'id':[3566841,6541227,3512441],'sex':[1,2,2],'level':[3,1,2]})

id_data = df2.values[:,0]
transtion_data = df2.values[:,1:]
enc = OneHotEncoder()
new_data = enc.fit_transform(transtion_data).toarray()
new_df = pd.concat((pd.DataFrame(id_data),pd.DataFrame(new_data)),axis = 1)
print(new_df)

'''
        id  level_high  level_low  level_middel  sex_male  sex_Female
0  3566841        True      False         False      True       False
1  6541227       False       True         False     False        True
2  3512441       False      False          True     False        True
         0    0    1    2    3    4
0  3566841  0.0  0.0  1.0  1.0  0.0
1  6541227  1.0  0.0  0.0  0.0  1.0
2  3512441  0.0  1.0  0.0  0.0  1.0

'''

降维

from sklearn.tree import DecisionTreeClassifier
from sklearn.decomposition import PCA

data = numpy.loadtxt('data.txt')
x = data[:,:-1]
y = data[:,-1] #标签
model_tree = DecisionTreeClassifier(random_state = 0)
model_tree.fit(x,y)
feature_importance = model_tree.feature_importances_#输出的是特征的重要性

#利用PCA进行将为
model_pca = PCA()
model_pca.fit(x)
model_pca.transform(x)
component = model_pca.components_
component_var = model_pca.explained_variance_
component_var_ratio = model_pca.explained_variance_ratio_

#相关性的分析
data = np.loadtxt('data5.txt',delimiter = '\t')
x = data[:,:-1]
correlation_matrix = np.corrcoef(x,rowvar = 0) #计算每列的相关性
print(correlation_matrix.round(2)) #保留两位小数

'''
正态分布的z-score,
就是将数据转化成均值以0为均值，1为方差的数据，即用数据减去列的均值，然后除以方差，
是一中中心化的方法，会改变原来的数据的结构，不适用于对于稀疏数据的处理 因此稀疏的数据不能运用中心化的方法

实现数据归一化的Max-Min
即数据减去所在列的均值，除以（所在列的最大值减去最小值）使得数据全部落在[0,1]区间，同时还能保留原有的数据结构

用于稀疏数据的MaxAbs 
即数据除以所在列绝对值最大的数，转换成了区间为[-1,1]的数据，也很好的保留了数据的原始的结构，可以运用到系数数据，稀疏矩阵CSR或CSC
CSR为行压缩的矩阵，CSC为列压缩的矩阵
'''
import numpy as np
import matplotlib.pyplot as plt
from sklearn import preprocessing #对数据预处理的函数

data = np.loadtxt('data6.txt')

#标准化处理
stand  = preprocessing.StandardScaler()
z_score_data = stand.fit_transform(data)

min_max = preprocessing.MinMaxScaler()
minmax_data = min_max.fit_transform(data)

abs_max = preprocessing.MaxAbsScaler()
abs_max_data = abs_max.fit_transform(data)

#绘制数据
all_data = [data,z_score_data,minmax_data,abs_max_data]
colors = ['red','blue','yellow','green']
markers = ['o','+','s','p']
scale_size = [5,10,12,15]
titles = ['data','z_score','minmax_data','abs_data']

for i,data_single in enumerate(all_data):
    plt.subplot(2,3,i+1)
    plt.scatter(data_single[:,:-1],data_single[:,-1],c = colors[i],marker = markers[i],s = scale_size[i])
    plt.title(titles[i])
plt.suptitle('show preprocessing data') #设置总的标题
plt.show()

#数据的离散化处理 针对时间的离散化
data = pd.read_table('data7.txt',names = ['id','amount','income','datatime','age']) #读取数据
for i,single_data in enumerate(data['datatime']):
    single_data_tmp = pd.to_datetime(single_data)
    data['datatime'][i] = single_data_tmp.weekday()#离散化为周几


#针对多值数据的离散化
map_df = pd.DataFrame([['0-10','0-40'],['10-20','0-40'],['20-30','0-40'],
                       ['30-40','0-40'],['40-50','40-80'],['50-60','40-80'],
                       ['60-70','40-80'],['70-80','40-80'],['80-90','>80'],['>90','>80']],columns = ['age','age2'])
data_df = data.merge(map_df,left_on = 'age',right_on = 'age',how = 'inner') #数据框关联匹配
new_data = data_df.drop('age',1)


#针对连续性的数据的离散化
#方法一 自定义分象的区间实现离散化
bins = [0,200,1000,5000,10000]
data['amount1'] = pd.cut(data['amount'],bins = bins)  #看看属于那个区间（1000，5000）or （0，200）
#方法二使用聚类实现
data_amount = data['amount']
data_reshape = data_amount.reshape((data_amount.shape[0],1)) #转换数据的形状
model_kmean = KMeans(n_clusters = 4,random_state = 0)
kmean_result = model_kmean.fit_predict(data_reshape) #使用聚类进行分
data['amount2'] = kmean_result

#针对连续性的数据的二值化

bin = preprocessing.Binarizer(threshold = data['income'].mean())
bin_data = bin.fit_transform(data['income'])
bin_data.resize(data['income'].shape)
print(bin_data)