import pandas as pd import numpy as np from sklearn.preprocessing import Imputer #数据预处理的函数 #缺失值处理 np.random.seed(8) df = pd.DataFrame(np.random.randn(6,4),columns = ['col1','col2','col3','col4']) df.iloc[1:2,1] = np.nan df.iloc[4,3] = np.nan #print(df.isnull()) #查看是否有缺失值,是缺失值返回True不是则返回False nan_row = df.isnull().any()#返回有缺失值的列 df.isnull().all() #返回所有的列 df2 = df.dropna() #去除所有的缺失值的行 nan_mean = Imputer(missing_values = 'NaN',strategy = 'mean',axis = 0) #将缺失值用列的均值填充 nan_result = nan_mean.fit_transform(df) nan_back = df.fillna(method = 'backfill',limit = 1) #用后面的值填充缺失值 限制的次数是一次 nan_pad = df.fillna(method = 'pad') #用前面的数填充缺失值 nan0 = df.fillna(0) #用0填充缺失值
#异常值处理 df =pd.DataFrame({'col1':[1,120,3,5,2,12,13], 'col2':[12,17,31,53,22,32,43]}) cols = df.columns df_zscore = df.copy() for col in cols: df_col = df[col] #每列的值 z_score = (df_col - df_col.mean())/df_col.std() #标准化处理超过预值的为异常值 df_zscore[col] = z_score.abs() > 2.0 print(df_zscore)
#重复值处理 data1 = ['a',3] data2 = ['b',2] data3 = ['a',3] data4 = ['c',2] data = pd.DataFrame([data1,data2,data3,data4],columns = ['col1','col2']) isduplicated = data.duplicated() #判断是否重复 是返回True否返回False new_data1 = data.drop_duplicates() #删除重复的值 new_data2 = data.drop_duplicates(['col1']) #删除col1列重复的 new_data3 = data.drop_duplicates(['col1','col2']) #删除这两列重复的值 print(new_data1)
from sklearn.preprocessing import OneHotEncoder #数据预处理的函数 data = pd.DataFrame({'id':[3566841,6541227,3512441],'sex':['male','Female','Female'],'level':['high','low','middel']}) data_new = data.copy() for col_num,col_name in enumerate(data): #和data.columns 一样 获取每个类的序号与列名 col_data = data[col_name] col_data_type = col_data.dtype if col_data_type == 'object': #非数值型的 data_new = data_new.drop(col_name,axis = 1) #删除需要转换的列 value_sets = col_data.unique() #出现的不同的值 for value_unique in value_sets: col_new_name = col_name+'_' + value_unique col_temp = data.iloc[:,col_num] new_col = (col_temp == value_unique) data_new[col_new_name] = new_col print(data_new) df2 = pd.DataFrame({'id':[3566841,6541227,3512441],'sex':[1,2,2],'level':[3,1,2]}) id_data = df2.values[:,0] transtion_data = df2.values[:,1:] enc = OneHotEncoder() new_data = enc.fit_transform(transtion_data).toarray() new_df = pd.concat((pd.DataFrame(id_data),pd.DataFrame(new_data)),axis = 1) print(new_df) ''' id level_high level_low level_middel sex_male sex_Female 0 3566841 True False False True False 1 6541227 False True False False True 2 3512441 False False True False True 0 0 1 2 3 4 0 3566841 0.0 0.0 1.0 1.0 0.0 1 6541227 1.0 0.0 0.0 0.0 1.0 2 3512441 0.0 1.0 0.0 0.0 1.0 '''
降维
from sklearn.tree import DecisionTreeClassifier from sklearn.decomposition import PCA data = numpy.loadtxt('data.txt') x = data[:,:-1] y = data[:,-1] #标签 model_tree = DecisionTreeClassifier(random_state = 0) model_tree.fit(x,y) feature_importance = model_tree.feature_importances_#输出的是特征的重要性 #利用PCA进行将为 model_pca = PCA() model_pca.fit(x) model_pca.transform(x) component = model_pca.components_ component_var = model_pca.explained_variance_ component_var_ratio = model_pca.explained_variance_ratio_
#相关性的分析 data = np.loadtxt('data5.txt',delimiter = '\t') x = data[:,:-1] correlation_matrix = np.corrcoef(x,rowvar = 0) #计算每列的相关性 print(correlation_matrix.round(2)) #保留两位小数
''' 正态分布的z-score, 就是将数据转化成均值以0为均值,1为方差的数据,即用数据减去列的均值,然后除以方差, 是一中中心化的方法,会改变原来的数据的结构,不适用于对于稀疏数据的处理 因此稀疏的数据不能运用中心化的方法 实现数据归一化的Max-Min 即数据减去所在列的均值,除以(所在列的最大值减去最小值)使得数据全部落在[0,1]区间,同时还能保留原有的数据结构 用于稀疏数据的MaxAbs 即数据除以所在列绝对值最大的数,转换成了区间为[-1,1]的数据,也很好的保留了数据的原始的结构,可以运用到系数数据,稀疏矩阵CSR或CSC CSR为行压缩的矩阵,CSC为列压缩的矩阵 ''' import numpy as np import matplotlib.pyplot as plt from sklearn import preprocessing #对数据预处理的函数 data = np.loadtxt('data6.txt') #标准化处理 stand = preprocessing.StandardScaler() z_score_data = stand.fit_transform(data) min_max = preprocessing.MinMaxScaler() minmax_data = min_max.fit_transform(data) abs_max = preprocessing.MaxAbsScaler() abs_max_data = abs_max.fit_transform(data) #绘制数据 all_data = [data,z_score_data,minmax_data,abs_max_data] colors = ['red','blue','yellow','green'] markers = ['o','+','s','p'] scale_size = [5,10,12,15] titles = ['data','z_score','minmax_data','abs_data'] for i,data_single in enumerate(all_data): plt.subplot(2,3,i+1) plt.scatter(data_single[:,:-1],data_single[:,-1],c = colors[i],marker = markers[i],s = scale_size[i]) plt.title(titles[i]) plt.suptitle('show preprocessing data') #设置总的标题 plt.show()
#数据的离散化处理 针对时间的离散化 data = pd.read_table('data7.txt',names = ['id','amount','income','datatime','age']) #读取数据 for i,single_data in enumerate(data['datatime']): single_data_tmp = pd.to_datetime(single_data) data['datatime'][i] = single_data_tmp.weekday()#离散化为周几 #针对多值数据的离散化 map_df = pd.DataFrame([['0-10','0-40'],['10-20','0-40'],['20-30','0-40'], ['30-40','0-40'],['40-50','40-80'],['50-60','40-80'], ['60-70','40-80'],['70-80','40-80'],['80-90','>80'],['>90','>80']],columns = ['age','age2']) data_df = data.merge(map_df,left_on = 'age',right_on = 'age',how = 'inner') #数据框关联匹配 new_data = data_df.drop('age',1) #针对连续性的数据的离散化 #方法一 自定义分象的区间实现离散化 bins = [0,200,1000,5000,10000] data['amount1'] = pd.cut(data['amount'],bins = bins) #看看属于那个区间(1000,5000)or (0,200) #方法二使用聚类实现 data_amount = data['amount'] data_reshape = data_amount.reshape((data_amount.shape[0],1)) #转换数据的形状 model_kmean = KMeans(n_clusters = 4,random_state = 0) kmean_result = model_kmean.fit_predict(data_reshape) #使用聚类进行分 data['amount2'] = kmean_result
#针对连续性的数据的二值化 bin = preprocessing.Binarizer(threshold = data['income'].mean()) bin_data = bin.fit_transform(data['income']) bin_data.resize(data['income'].shape) print(bin_data)