[AI 笔记] Softmax

参考：
CS231n课程

Softmax

Softmax，亦称Multinomial Logistic Regression，与SVM一样，用于计算分类损失。

Softmax可以理解为将线性分类中，线性函数的得分归一化成概率，而后依据概率计算损失，同时在分类时，分类结果为概率最高的类别。

其公式如下图所示：

其中 s 即为线性函数的得分，其下标指代某一类别；P 为当样本为 xi 时，将其分类为 k 的概率，P 右边的式子即为归一化函数，将得分归一化为概率。

而Softmax的损失，即图中 Li，为当样本为 xi 时，将其正确归类为其标签 yi 的概率的对数的相反数。

则训练损失为 Li 对其下标 i 的累加。

下图形象地展示了Softmax计算损失的过程。

代码实现

Softmax 对 cifar-10 进行分类。

#SoftMax_object.py
import numpy as np
class SoftMax(object):

    def __init__(self):
        self.W = None
        self.reg = 1e-5
        self.num_classes = 10
        self.learning_rate = 1e-3

    def softmax_loss_vectorized(self, W, X, y, reg):
        # 初始化参数
        loss = 0.0
        dW = np.zeros_like(W)
        num_train = X.shape[0]    # N
        scores = X.dot(W)         # N×10
        shift_scores = scores - np.max(scores, axis=1).reshape(-1, 1)  # 减去每一行最大值
        softmax_output = np.exp(
            shift_scores)/np.sum(np.exp(shift_scores), axis=1).reshape(-1, 1)  # 归一化概率
        # 计算损失
        loss = -np.sum(np.log(softmax_output[range(num_train), list(y)]))
        loss /= num_train
        loss += 0.5 * reg * np.sum(W*W)  # 正则化

        dS = softmax_output.copy()
        dS[range(num_train), list(y)] += -1
        dW = (X.T).dot(dS)
        dW = dW / num_train + reg * W  # 计算梯度
        return loss, dW

    def train(self, X, y, num_classes=10, learning_rate=2.0e-7, reg=9.0e+3, num_iters=1000,
              batch_size=200, verbose=False):
        """
         使用随机梯度下降来训练这个分类器
         输入：
         -X ：一个numpy数组，维数为（N,D）
         -Y ： 一个numpy数组，维数为（N，）
         -learning rate: float ，优化的学习率
         -reg : float，正则化强度
         -num_iters: integer, 优化时训练的步数
         -batch_size：integer, 每一步使用的训练样本数
         -ver bose : boolean， 若为真，优化时打印过程

         输出：
         一个存储每次训练的损失函数值的List
        """
        num_train, dim = X.shape
        self.num_classes = num_classes

        if self.W is None:
            # 简易初始化W
            self.W = 0.001 * np.random.randn(dim, num_classes)

        # 使用随机梯度下降优化W
        loss_history = []
        for it in range(num_iters):
            X_batch = None
            y_batch = None
            """
            从训练集中采样batch_size个样本和对应的标签，在这一轮梯度下降中使用。
            把数据存储在X_batch中，把对应的标签存储在y_batch中
            采样后，X_batch的形状为（dim,batch_size)，y_batch的形状为（batch_size,）
            """
            batch_inx = np.random.choice(num_train, batch_size)
            X_batch = X[batch_inx, :]
            y_batch = y[batch_inx]

            loss, grad = self.softmax_loss_vectorized(
                self.W, X_batch, y_batch, reg)
            loss_history.append(loss)

            """
            使用梯度和学习率更新权重
            """
            self.W = self.W - learning_rate * grad
            if verbose and it % 10 == 0:
                print('iteration %d / %d: loss %f' % (it, num_iters, loss))

        return loss_history

    def predict(self, X):

        y_pred = np.zeros(X.shape[0])
        scores = X.dot(self.W)
        y_pred = np.argmax(scores, axis=1)
        return y_pred

#SoftMax_run.py
from SoftMax_object import SoftMax
import time
import numpy as np
#import matplotlib.pyplot as plt


def unpickle(file):
    import pickle
    with open(file, 'rb') as fo:
        dict = pickle.load(fo, encoding='bytes')
    return dict


# get the training data
dataTrain = []
labelTrain = []
for i in range(1, 6):
    #dic = unpickle("/home/project/data/ZhouZhenKang/cifar-10-batches-py/data_batch_"+str(i))
    dic = unpickle(
        "D:/Desktop/论文及算法实现/DATA_SETS/cifar-10-batches-py/data_batch_"+str(i))
    for item in dic[b"data"]:
        # print(item.shape)
        #item = item.tolist()
        # item.append(1)
        dataTrain.append(item)
    for item in dic[b"labels"]:
        labelTrain.append(item)

# get test data
dataTest = []
labelTest = []
dic = unpickle("D:/Desktop/论文及算法实现/DATA_SETS/cifar-10-batches-py/test_batch")
#dic = unpickle("/home/project/data/ZhouZhenKang/cifar-10-batches-py/test_batch")
for item in dic[b"data"]:
    #item = item.tolist()
    # item.append(1)
    dataTest.append(item)
for item in dic[b"labels"]:
    labelTest.append(item)


dataTr = np.asarray(dataTrain)
dataTs = np.asarray(dataTest)
labelTr = np.asarray(labelTrain)
labelTs = np.asarray(labelTest)
print('dataTr.shape: {}'.format(dataTr.shape))


softmax = SoftMax()
tic = time.time()
loss_hist = softmax.train(dataTr, labelTr, num_classes=10, learning_rate=2.0e-7, reg=9.0e+3,
                          num_iters=1000, batch_size=1000, verbose=True)
toc = time.time()
print('That took %fs' % (toc - tic))

# A useful debugging strategy is to plot the loss as a function of
# iteration number:
'''
plt.plot(loss_hist)
plt.xlabel('Iteration number')
plt.ylabel('Loss value')
plt.show()
'''

# Write the SVM.predict function and evaluate the performance on both the
# training and validation set
y_train_pred = softmax.predict(dataTr)
print('training accuracy: %f' % (np.mean(labelTr == y_train_pred), ))
y_val_pred = softmax.predict(dataTs)
print('validation accuracy: %f' % (np.mean(labelTs == y_val_pred), ))

iteration 950 / 1000: loss 6.313303
iteration 960 / 1000: loss 6.184987
iteration 970 / 1000: loss 6.014891
iteration 980 / 1000: loss 5.860609
iteration 990 / 1000: loss 5.765472
That took 37.708521s
training accuracy: 0.346120
validation accuracy: 0.339200