2017cs231n assignment1(softmax)

softmax.py

$$L_i=-log(\frac{e^s{y}_i}{{\sum }_j{e}^{s}j})$$

其中$s=f(x_i,W)$

def softmax_loss_naive(W, X, y, reg):
    """
    Softmax损失函数, 朴素实现 (有循环)

    输入有维度D, 有C个分类,我们对N个样例进行操作

    Inputs:
    - W: 保存权重的numpu数组，形状为(D, C) 
    - X: 保存数据迷你表的numpy数组，形状为(N, D) 
    - y: 保存训练标签的numpu数组，形状为(N,) ; y[i] = c 表示X[i]标签为c, where 0 <= c < C.
    - reg: (float) 正则化强度

    Returns a tuple of:
    - loss as single float
    - gradient with respect to weights W; an array of same shape as W
    """
    # 损失和梯度初始化为0
    loss = 0.0
    dW = np.zeros_like(W)
    
    num_classes = W.shape[1]  
    num_train = X.shape[0]  
    for i in range(num_train):
        scores = X[i].dot(W)  
        scores = scores - np.max(scores)  
        scores_exp = np.exp(scores)     # 指数操作
        ds_w = np.repeat(X[i], num_classes).reshape(-1, num_classes)   # 计算得分对权重的倒数
        scores_exp_sum = np.sum(scores_exp)
        pk = scores_exp[y[i]] / scores_exp_sum
        loss += -np.log(pk)   
        dl_s = np.zeros(W.shape)  # 开始计算loss对得分的倒数
        for j in range(num_classes):
            if j == y[i]:
                dl_s[:, j] = pk - 1    # 对于输入正确分类的那一项，倒数与其他不同
            else:
                dl_s[:, j] = scores_exp[j] / scores_exp_sum
        dW_i = ds_w * dl_s
        dW += dW_i
    loss /= num_train
    dW /= num_train
    loss += reg * np.sum(W * W)
    dW += W * 2 * reg

    return loss, dW

def softmax_loss_vectorized(W, X, y, reg):
    """
    Softmax损失函数, 全向量化操作.

    """
    # 损失和梯度初始化为0
    loss = 0.0
    dW = np.zeros_like(W)

    #############################################################################
    # TODO: Compute the softmax loss and its gradient using no explicit loops.  #
    # Store the loss in loss and the gradient in dW. If you are not careful     #
    # here, it is easy to run into numeric instability. Don't forget the        #
    # regularization!                                                           #
    #############################################################################
    # *****START OF YOUR CODE (DO NOT DELETE/MODIFY THIS LINE)*****

    num_classes = W.shape[1]
    num_train = X.shape[0]
    scores = X.dot(W)
    scores = scores - np.max(scores, 1, keepdims=True)
    scores_exp = np.exp(scores)
    sum_s = np.sum(scores_exp, 1, keepdims=True)
    p = scores_exp / sum_s
    loss = np.sum(-np.log(p[np.arange(num_train), y]))

    ind = np.zeros_like(p)
    ind[np.arange(num_train), y] = 1
    dW = X.T.dot(p - ind)

    loss /= num_train
    dW /= num_train
    loss += reg * np.sum(W * W)
    dW += W * 2 * reg
 

    return loss, dW

softmax.ipynb

# 使用验证集去优化超参数(正则化强度和学习率).应该使用不同范围的正则化强度和学习率来进行实验;
# 如果你很仔细应该可以发现这个分类器在验证集的准确率可以超过0.35
from cs231n.classifiers import Softmax
results = {}
best_val = -1
best_softmax = None
learning_rates = [1e-7, 5e-7]
regularization_strengths = [2.5e4, 5e4]

for lr in learning_rates:
    for reg in regularization_strengths:
        softmax = Softmax()
        loss_hist = softmax.train(X_train, y_train, lr, reg,
                      num_iters=500, verbose=True)
        y_train_pred = softmax.predict(X_train)
        acc_tr = np.mean(y_train == y_train_pred)
        y_val_pred = softmax.predict(X_val)
        acc_val = np.mean(y_val == y_val_pred)
        results[(lr, reg)] = (acc_tr, acc_val)
        if best_val < acc_val:
            best_val = acc_val
            best_softmax = softmax
            
# 打印结果
for lr, reg in sorted(results):
    train_accuracy, val_accuracy = results[(lr, reg)]
    print('lr %e reg %e train accuracy: %f val accuracy: %f' % (
                lr, reg, train_accuracy, val_accuracy))
    
print('best validation accuracy achieved during cross-validation: %f' % best_val)

iteration 0 / 500: loss 767.707658
iteration 100 / 500: loss 281.938693
iteration 200 / 500: loss 104.477796
iteration 300 / 500: loss 39.464869
iteration 400 / 500: loss 15.802784
iteration 0 / 500: loss 1535.985414
iteration 100 / 500: loss 206.705699
iteration 200 / 500: loss 29.451702
iteration 300 / 500: loss 5.817203
iteration 400 / 500: loss 2.630755
iteration 0 / 500: loss 774.900384
iteration 100 / 500: loss 6.912728
iteration 200 / 500: loss 2.187072
iteration 300 / 500: loss 2.108355
iteration 400 / 500: loss 2.067168
iteration 0 / 500: loss 1518.743050
iteration 100 / 500: loss 2.188737
iteration 200 / 500: loss 2.113567
iteration 300 / 500: loss 2.156632
iteration 400 / 500: loss 2.138307
lr 1.000000e-07 reg 2.500000e+04 train accuracy: 0.311714 val accuracy: 0.315000
lr 1.000000e-07 reg 5.000000e+04 train accuracy: 0.312408 val accuracy: 0.323000
lr 5.000000e-07 reg 2.500000e+04 train accuracy: 0.317265 val accuracy: 0.329000
lr 5.000000e-07 reg 5.000000e+04 train accuracy: 0.317755 val accuracy: 0.323000
best validation accuracy achieved during cross-validation: 0.329000