【Machine Learning 学习笔记】feature engineering中noisy feature的影响
通过本篇博客记录一下添加噪声对Lasso和SVM的影响,采用的数据集为sklearn的make_moons。
数据准备
import numpy as np
import matplotlib.pyplot as plt
from sklearn.svm import SVC
from sklearn.linear_model import Lasso
from sklearn.model_selection import KFold
from sklearn.metrics import accuracy_score
from sklearn.datasets import make_moons
n = 300
n_tr = 200
X, y = make_moons(n, shuffle=True, noise=0.2, random_state=112)
y[y==0] = -1
# standardise the data
trmean = np.mean(X[:n_tr, :], axis=0)
trvar = np.var(X[:n_tr, :], axis=0)
X = (X - trmean[np.newaxis, :]) / np.sqrt(trvar)[np.newaxis, :]
# take first n_tr as training, others as test.
Xtr = X[:n_tr, :]
ytr = y[:n_tr]
Xt = X[n_tr:, :]
yt = y[n_tr:]
# inspect the dataset visually
plt.figure()
plt.scatter(Xtr[:, 0], Xtr[:, 1], c=ytr, marker='x')
plt.scatter(Xt[:, 0], Xt[:, 1], c=yt, marker='o')
plt.show()
数据集如图所示:
![](https://i-blog.csdnimg.cn/blog_migrate/5ad32f89b6229e40b185a58804023ca5.png)
需要通过grid search和k fold cross validation进行最优参数寻找,数据准备有:
k = 5 # k fold cross validation
# the parameters to consider in cross-validation
# order lasso params from largest to smallest so that if ties occur CV will select the one with more sparsity
params_lasso = [1, 1e-1, 1e-2, 1e-3, 1e-4]
params_rbf = [1e-3, 1e-2, 1e-1, 1, 10, 100]
params_svm = [1e-3, 1e-2, 1e-1, 1, 10, 100, 1000] # smaller C: more regularisation
# the noisy features to be considered; for n noisy features, add the first n columns
n_tot_noisy_feats = 50
m = 9
np.random.seed(92)
X_noise = 2*np.random.randn(X.shape[0], n_tot_noisy_feats)
Xtr_noise = X_noise[:n_tr, :]
Xt_noise = X_noise[n_tr:, :]
svm_accs = []
lasso_accs = []
lasso_n_feats = []
# do CV over the parameters
amount_of_noisy_features = np.arange(n_tot_noisy_feats+1) # include both 0 and 50
for n_noisy_features in amount_of_noisy_features:
print('number of noisey feature: {}'.format(n_noisy_features))
Xtr_noisy = np.copy(Xtr)
Xt_noisy = np.copy(Xt)
if n_noisy_features > 0:
Xtr_noisy = np.hstack((Xtr_noisy, Xtr_noise[:, :n_noisy_features]))
Xt_noisy = np.hstack((Xt_noisy, Xt_noise[:, :n_noisy_features]))
lasso_validation_accs = np.zeros((k, len(params_lasso)))
svm_validation_accs = np.zeros((k, len(params_rbf), len(params_svm)))
# make the training-validation splits on the training data
kf = KFold(n_splits=k)
fold_index = 0
for train_index, validation_index in kf.split(Xtr_noisy):
# now then get the results on all the parameters considered
# SVM with the two parameters
for g_index, gamma in enumerate(params_rbf):
for s_index, svm_param in enumerate(params_svm):
svm = SVC(C=svm_param, gamma=gamma)
svm.fit(Xtr_noisy[train_index, :], ytr[train_index])
preds = svm.predict(Xtr_noisy[validation_index])
svm_validation_accs[fold_index, g_index, s_index] = accuracy_score(ytr[validation_index], preds)
# lasso with the one parameter
for l_index, lasso_param in enumerate(params_lasso):
lasso = Lasso(alpha=lasso_param)
lasso.fit(Xtr_noisy[train_index, :], ytr[train_index])
preds = np.sign(lasso.predict(Xtr_noisy[validation_index]))
lasso_validation_accs[fold_index, l_index] = accuracy_score(ytr[validation_index], preds)
fold_index += 1
选择最优的参数并进行拟合、测试:
# average results over folds
lasso_validation_avg = np.mean(lasso_validation_accs, axis=0)
svm_validation_avg = np.mean(svm_validation_accs, axis=0)
# get the indexes of the best parameters and get the parameters at these indices
best_lasso_param = params_lasso[np.argmax(lasso_validation_avg)]
[best_svm_gamma_index, best_svm_C_index] = np.unravel_index(np.argmax(svm_validation_avg, axis=None), svm_validation_avg.shape)
best_svm_gamma = params_rbf[best_svm_gamma_index]
best_svm_C = params_svm[best_svm_C_index]
print("best lasso parameter is: {}".format(best_lasso_param))
print("best svm gamma parameter is: {}".format(best_svm_gamma))
print("best svm C parameter is: {}".format(best_svm_C))
# train with the whole training data with the chosen parameters
svm = SVC(C=best_svm_C, gamma=best_svm_gamma)
svm.fit(Xtr_noisy, ytr)
svm_preds = svm.predict(Xt_noisy)
svm_accs.append(accuracy_score(yt, svm_preds))
lasso = Lasso(alpha=best_lasso_param)
lasso.fit(Xtr_noisy, ytr)
lasso_preds = np.sign(lasso.predict(Xt_noisy))
lasso_accs.append(accuracy_score(yt, lasso_preds))
# how many features did lasso select?
lasso_n_feats.append(np.count_nonzero(lasso.coef_))
绘制结果图
plt.figure()
plt.plot(amount_of_noisy_features, svm_accs, label='svm')
plt.plot(amount_of_noisy_features, lasso_accs, label='lasso')
plt.title('cross validation')
plt.legend()
svm_accs = np.array(svm_accs)
lasso_accs = np.array(lasso_accs)
print('lasso was better than svm for the first time with {} noisy features'.format(np.where(svm_accs - lasso_accs<0)[0][0]))
plt.figure()
plt.plot(amount_of_noisy_features, lasso_n_feats)
plt.title('lasso features')
plt.show()
![](https://i-blog.csdnimg.cn/blog_migrate/ccdbac2f2681decf7f9df63845f4afcc.png)
![](https://i-blog.csdnimg.cn/blog_migrate/834b2c2897b784f3f5e44db25b7ddae4.png)
可以看出,当noisy feature的数目很小时,SVM的准确率高于Lasso,但是当noisy feature的数目变多后Lasso的结果反而更好。