正在代码理解中
softman.py
import numpy as np
from random import shuffle
def softmax_loss_naive(weight , X , y, reg):
"""用循环实现softmax损失函数 D,C,N分别表示数据维度，标签种类个数和数据批大小
Inputs: - weight (D, C)：weights. - X (N, D)：data. - y (N,)： labels - reg: (float) regularization strength
Returns : - loss - gradient """
loss = 0.0
dweight = np.zeros_like(weight)
num_classes = weight.shape[1]
num_train = X.shape[0]
for i in range(num_train):
scores = np.dot(X[i],weight)
shift_scores = scores-max(scores)
dom = np.log(np.sum(np.exp(shift_scores)))
loss_i = -shift_scores[y[i]]+dom
loss += loss_i
for j in range(num_classes):
softmax_output = np.exp(shift_scores[j])/sum(np.exp(shift_scores))
if j == y[i]:
dweight[:, j] += (-1 + softmax_output) * X[i].T
else:
dweight[:,j] += softmax_output * X[i].T
loss /= num_train
loss += reg * np.sum(weight * weight)
dweight = dweight/num_train + 2*reg*weight
return loss, dweight
def softmax_loss_vectorized(weight, X, y, reg):
"""无循环的实现"""
loss = 0.0
dweight = np.zeros_like(weight)
num_classes = weight.shape[1]
num_train = X.shape[0]
scores = np.dot(X,weight)
shift_scores = scores-np.max(scores,axis=1).reshape(-1,1)
softmax_output = np.exp(shift_scores)/np.sum(np.exp(shift_scores), axis = 1).reshape(-1,1)
loss=np.sum(-np.log(softmax_output[range(num_train),y]))
loss=loss/num_train+reg * np.sum(weight * weight)
dweight = softmax_output.copy()
dweight[range(num_train), y] -= 1
dweight=np.dot(X.T,dweight)
dweight = dweight/num_train + 2*reg*weight
return loss, dweight
linear_svm.py
import numpy as np
from random import shuffle
def svm_loss_naive(weight, X, y, reg):
"""用循环实现的SVM loss计算 这里的loss函数使用的是margin loss
Inputs: - weight (D, C)： 权重矩阵. - X (N, D)： 批输入 - y (N,) 标签 - reg: 正则参数
Returns : - loss float - weight的梯度 """
dweight =np.zeros(weight.shape)
num_classes = weight.shape[1]
num_train =X.shape[0]
loss = 0.0
for i in range(num_train):
scores = X[i].dot(weight)
correct_class_score =scores[y[i]]
for j in range(num_classes):
if j ==y[i]:
continue
margin = scores[j] - correct_class_score + 1
if margin >0:
loss +=margin
dweight[:,j]+=X[i].T
dweight[:,y[i]]-=X[i].T
loss /=num_train
dweight/=num_train
loss += reg * np.sum(weight * weight)
dweight+=2* reg * weight
return loss, dweight
def svm_loss_vectorized(weight, X, y, reg):
"""不使用循环，利用numpy矩阵运算的特性实现loss和梯度计算51 """
loss = 0.0
dweight =np.zeros(weight.shape)
#计算loss
num_classes = weight.shape[1]
num_train =X.shape[0]
scores=np.dot(X,weight)#得到得分矩阵(N，C)
correct_socre=scores[range(num_train), list(y)].reshape(-1,1)#得到每个输入的正确分类的分数
margins=np.maximum(0,scores-correct_socre+1)
margins[range(num_train), list(y)] =0
loss=np.sum(margins)/num_train+reg * np.sum(weight * weight)
#计算梯度
mask=np.zeros((num_train,num_classes))
mask[margins>0]=1
mask[range(num_train),list(y)]-=np.sum(mask,axis=1)
dweight=np.dot(X.T,mask)
dweight/=num_train
dweight+=2* reg * weight
return loss, dweight
linear_classfication.py
import numpy as np
from linear_svm import *
from softmax import *
class LinearClassifier(object):#线性分类器的基类
def __init__(self):
self.weight =None
def train(self, X, y, learning_rate=1e-3, reg=1e-5, num_iters=100, batch_size=200, verbose=False):
"""使用SGD优化参数矩阵
Inputs: - X (N, D) - y (N,) - learning_rate: 学习率. - reg: 正则参数. - num_iters: (int) 训练迭代的次数
- batch_size: (int) 每次迭代使用的样本数量. - verbose: (boolean) 是否显示训练进度
Outputs: 返回一个list保存了每次迭代的loss """
num_train, dim =X.shape
num_classes = np.max(y) + 1 #假设有k类，y的取值为【0，k-1】且最大的下标一定会在训练数据中出现
#初始化权重矩阵
if self.weight is None:
self.weight = 0.001 *np.random.randn(dim, num_classes)
loss_history =[]
for it in range(num_iters): #在每次迭代，随机选择batch_size个数据
mask=np.random.choice(num_train,batch_size,replace=True)
X_batch =X[mask]
y_batch =y[mask]
#计算损失和梯度
loss, grad =self.loss(X_batch, y_batch, reg)
loss_history.append(loss)
#更新参数
self.weight -= grad* learning_rate
if verbose and it % 100 ==0:
print('iteration %d / %d: loss %f' %(it, num_iters, loss))
return loss_history
def predict(self, X):
""" 使用训练好的参数来对输入进行预测60
Inputs: - X (N, D) Returns: - y_pred (N，):预测的正确分类的下标 """
y_pred=np.dot(X,self.W)
y_pred = np.argmax(y_pred, axis = 1)
return y_pred
def loss(self, X_batch, y_batch, reg):
"""这只是一个线性分类器的基类 不同的线性分类器loss的计算方式不同 所以需要在子类中重写 """
pass
class LinearSVM(LinearClassifier):
"""使用SVM loss"""
def loss(self, X_batch, y_batch, reg):
return svm_loss_vectorized(self.weight, X_batch, y_batch, reg)
class Softmax(LinearClassifier):
"""使用交叉熵"""
def loss(self, X_batch, y_batch, reg):
return softmax_loss_vectorized(self.weight, X_batch, y_batch, reg)
data.utils.py
#获取数据的部分
from six.moves import cPickle as pickle
import numpy as np
import os
from imageio import imread
import platform
def load_pickle(f):
version =platform.python_version_tuple()
if version[0] == '2':
return pickle.load(f)
elif version[0] == '3':
return pickle.load(f, encoding='latin1')
raise ValueError("invalid python version: {}".format(version))
def load_CIFAR_batch(filename):
"""CIRAR的数据是分批的，这个函数的功能是载入一批数据"""
with open(filename, 'rb') as f:
datadict = load_pickle(f) #以二进制方式打开文件
X = datadict['data']
Y = datadict['labels']
X = X.reshape(10000, 3, 32, 32).transpose(0,2,3,1).astype("float")
Y =np.array(Y)
return X, Y
def load_CIFAR10(ROOT):
"""load 所有的数据"""
xs =[]
ys =[]
for b in range(1,6):
f = os.path.join(ROOT, 'data_batch_%d' %(b, ))
X, Y =load_CIFAR_batch(f)
xs.append(X)
ys.append(Y)
Xtr =np.concatenate(xs)
Ytr =np.concatenate(ys)
del X, Y
Xte, Yte = load_CIFAR_batch(os.path.join(ROOT, 'test_batch'))
return Xtr, Ytr, Xte, Yte
def get_CIFAR10_data (num_training=49000, num_validation=1000, num_test=1000, subtract_mean=True):
""" Load the CIFAR-10 dataset from disk and perform preprocessing to prepare
it for classifiers. These are the same steps as we used for the SVM, but condensed to a single function. """
cifar10_dir = 'C:/Users/Dell/Desktop/cv/cifar-10-python/cifar-10-batches-py' #Load the raw CIFAR-10 data
X_train, y_train, X_test, y_test =load_CIFAR10(cifar10_dir)
#Subsample the data
mask = list(range(num_training, num_training +num_validation))
X_val =X_train[mask]
y_val =y_train[mask]
mask =list(range(num_training))
X_train =X_train[mask]
y_train =y_train[mask]
mask =list(range(num_test))
X_test =X_test[mask]
y_test =y_test[mask]
#Normalize the data: subtract the mean image
if subtract_mean:
mean_image = np.mean(X_train, axis=0)
X_train -=mean_image
X_val -=mean_image
X_test -=mean_image
#Transpose so that channels come first
X_train = X_train.transpose(0, 3, 1, 2).copy()
X_val = X_val.transpose(0, 3, 1, 2).copy()
X_test = X_test.transpose(0, 3, 1, 2).copy()
#Package data into a dictionary
return { 'X_train': X_train, 'y_train': y_train, 'X_val': X_val, 'y_val': y_val, 'X_test': X_test, 'y_test': y_test,}
test.py
#实现hinge_loss和sotfmax_loss
import random
import numpy as np
from data_utils import load_CIFAR10
import matplotlib.pyplot as plt
from linear_svm import svm_loss_naive,svm_loss_vectorized
from softmax import softmax_loss_naive,softmax_loss_vectorized
import time
from linear_classfication import LinearSVM, Softmax
plt.rcParams['figure.figsize'] = (10.0, 8.0) #set default size of plots
plt.rcParams['image.interpolation'] = 'nearest'
plt.rcParams['image.cmap'] = 'gray'
###################################第一部分 载入数据并处理###############################
#载入CIFAR10数据.
cifar10_dir = 'C:/Users/Dell/Desktop/cv/cifar-10-python/cifar-10-batches-py'
X_train, y_train, X_test, y_test =load_CIFAR10(cifar10_dir)
print('Training data shape:', X_train.shape)
print('Training labels shape:', y_train.shape)
print('Test data shape:', X_test.shape)
print('Test labels shape:', y_test.shape)
#每个分类选几个图片显示观察一下
classes = ['plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck']
num_classes =len(classes)
samples_per_class = 7
for y, cls in enumerate(classes):
idxs = np.flatnonzero(y_train ==y)
idxs = np.random.choice(idxs, samples_per_class, replace=False)
for i, idx in enumerate(idxs):
plt_idx = i * num_classes + y + 1
plt.subplot(samples_per_class, num_classes, plt_idx)
plt.imshow(X_train[idx].astype('uint8'))
plt.axis('off')
if i ==0:
plt.title(cls)
plt.show()
#把数据分为训练集，验证集和测试集。
#用一个小子集做测验，运行更快。
num_training = 49000
num_validation = 1000
num_test = 1000
num_dev = 500
#数据集本身没有给验证集，需要自己把训练集分成两部分
mask = range(num_training, num_training +num_validation)
X_val =X_train[mask]
y_val =y_train[mask]
mask =range(num_training)
X_train =X_train[mask]
y_train =y_train[mask]
mask = np.random.choice(num_training, num_dev, replace=False)
X_dev =X_train[mask]
y_dev =y_train[mask]
mask =range(num_test)
X_test =X_test[mask]
y_test =y_test[mask]
print('Train data shape:', X_train.shape)
print('Train labels shape:', y_train.shape)
print('Validation data shape:', X_val.shape)
print('Validation labels shape:', y_val.shape)
print('Test data shape:', X_test.shape)
print('Test labels shape:', y_test.shape)
X_train = np.reshape(X_train, (X_train.shape[0], -1))
X_val = np.reshape(X_val, (X_val.shape[0], -1))
X_test = np.reshape(X_test, (X_test.shape[0], -1))
X_dev = np.reshape(X_dev, (X_dev.shape[0], -1))
print('Training data shape:', X_train.shape)
print('Validation data shape:', X_val.shape)
print('Test data shape:', X_test.shape)
print('dev data shape:', X_dev.shape)
#预处理: 把像素点数据化成以0为中心
#第一步: 在训练集上计算图片像素点的平均值
mean_image = np.mean(X_train, axis=0)
print(mean_image.shape)
plt.figure(figsize=(4,4))
plt.imshow(mean_image.reshape((32,32,3)).astype('uint8')) #可视化一下平均值
plt.show()
#第二步: 所有数据都减去刚刚得到的均值
X_train -=mean_image
X_val -=mean_image
X_test -=mean_image
X_dev -=mean_image
#第三步: 给所有的图片都加一个位，并设为1，这样在训练权重的时候就不需要b了，只需要w
#相当于把b的训练并入了W中
X_train = np.hstack([X_train, np.ones((X_train.shape[0], 1))])
X_val = np.hstack([X_val, np.ones((X_val.shape[0], 1))])
X_test = np.hstack([X_test, np.ones((X_test.shape[0], 1))])
X_dev = np.hstack([X_dev, np.ones((X_dev.shape[0], 1))])
print(X_train.shape, X_val.shape, X_test.shape, X_dev.shape)
###################################第二部分 定义需要用到的函数###########################
def cmp_naiveANDvectorized(naive,vectorized):
'''每个损失函数都用两种方式实现：循环和无循环(即利用numpy的特性)130 '''
W = np.random.randn(3073, 10) * 0.0001
#对比两张实现方式的计算时间
tic =time.time()
loss_naive, grad_naive = naive(W, X_dev, y_dev, 0.000005)
toc =time.time()
print('Naive computed in %fs' % ( toc -tic))
tic =time.time()
loss_vectorized, grad_vectorized = vectorized(W, X_dev, y_dev, 0.000005)
toc =time.time()
print('Vectorized computed in %fs' % ( toc -tic))
#检验损失的实现是否正确，对于随机初始化的数据的权重，
#softmax_loss应该约等于-log(0.1),svm_loss应该约等于9
print('loss %f %f' %(loss_naive , loss_vectorized))
#对比两种实现方式得到的结果是否相同
print('difference loss %f' % (loss_naive -loss_vectorized))
difference = np.linalg.norm(grad_naive - grad_vectorized, ord='fro')
print('difference gradient: %f' %difference)
def cross_choose(Linear_classifier,learning_rates,regularization_strengths):
'''选择超参数157 '''
results = {} #存储每一对超参数对应的训练集和验证集上的正确率
best_val = -1 #最好的验证集上的正确率
best_model = None #最好的验证集正确率对应的svm类的对象
best_loss_hist=None
for rs in regularization_strengths:
for lr in learning_rates:
classifier =Linear_classifier
loss_hist = classifier.train(X_train, y_train, lr, rs, num_iters=5)
y_train_pred =classifier.predict(X_train)
train_accuracy = np.mean(y_train ==y_train_pred)
y_val_pred =classifier.predict(X_val)
val_accuracy = np.mean(y_val ==y_val_pred)
if val_accuracy >best_val:
#print("1")
best_val =val_accuracy
best_model =classifier
best_loss_hist=loss_hist
results[(lr,rs)] =train_accuracy, val_accuracy
for lr, reg in sorted(results):
train_accuracy, val_accuracy =results[(lr, reg)]
print ('lr %e reg %e train accuracy: %f val accuracy: %f' %( lr, reg, train_accuracy, val_accuracy))
print('best validation accuracy achieved during cross-validation: %f' %best_val)
#可视化loss曲线
plt.plot(best_loss_hist)
plt.xlabel('Iteration number')
plt.ylabel('Loss value')
plt.show()
return results, best_val, best_model
def show_weight(best_model):#看看最好的模型的效果
#可视化学到的权重
y_test_pred = best_model.predict(X_test)
test_accuracy = np.mean(y_test ==y_test_pred)
print('final test set accuracy: %f' %test_accuracy)
w = best_model.weight[:-1,:] #去掉偏置参数
w = w.reshape(32, 32, 3, 10)
w_min, w_max =np.min(w), np.max(w)
classes = ['plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck']
for i in range(10):
plt.subplot(2, 5, i + 1)
#把权重转换到0-255
wimg = 255.0 * (w[:, :, :, i].squeeze() - w_min) / (w_max -w_min)
plt.imshow(wimg.astype('uint8'))
plt.axis('off')
plt.title(classes[i])
plt.show()
##########################第三部分 应用和比较svm_loss和softmax_loss######################
cmp_naiveANDvectorized(svm_loss_naive,svm_loss_vectorized)
learning_rates = [(1+i*0.1)*1e-7 for i in range(-3,5)]
regularization_strengths = [(5+i*0.1)*1e3 for i in range(-3,3)]
#正则参数的选择要根据正常损失和W*W的大小的数量级来确定，初始时正常loss大概是9，W*W大概是1e-6
#可以观察最后loss的值的大小来继续调整正则参数的大小，使正常损失和正则损失保持合适的比例
results,best_val,best_model=cross_choose(LinearSVM(),learning_rates,regularization_strengths)
show_weight(best_model)
print("--------------------------------------------------------")
cmp_naiveANDvectorized(softmax_loss_naive,softmax_loss_vectorized)
learning_rates = [(2+i*0.1)*1e-7 for i in range(-2,2)]
regularization_strengths = [(7+i*0.1)*1e3 for i in range(-3,3)]
results,best_val,best_model=cross_choose(Softmax(),learning_rates,regularization_strengths)
show_weight(best_model)
扫一扫
2918
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
