pytorch学习笔记——CrossEntropyLoss、NLLLoss、Softmax和LogSoftmax之间的关系

import torch.nn as nn
import torch

data = torch.Tensor([[-0.2733, 0.3222, 0.2605],
                     [1.5393, 1.1688, -0.0975],
                     [0.3943, 0.5172, -0.9425]])  # 一个3*3的矩阵
print(data)
'''
程序运行的一次结果
tensor([[-0.2733,  0.3222,  0.2605],
        [ 1.5393,  1.1688, -0.0975],
        [ 0.3943,  0.5172, -0.9425]])
'''
sm = nn.Softmax(dim=1)  # 按行 softmax
print(sm(data))
'''
程序运行的一次结果
tensor([[0.2213, 0.4014, 0.3774],
        [0.5305, 0.3663, 0.1032],
        [0.4178, 0.4725, 0.1098]])
'''
print(torch.log(sm(data)))  # log(softmax)
'''
程序运行的一次结果
tensor([[-1.5084, -0.9129, -0.9746],
        [-0.6339, -1.0044, -2.2707],
        [-0.8728, -0.7498, -2.2095]])
'''
slm = nn.LogSoftmax(dim=1)
print(slm(data))  # LogSoftmax
'''
程序运行的一次结果
tensor([[-1.5084, -0.9129, -0.9746],
        [-0.6339, -1.0044, -2.2707],
        [-0.8728, -0.7498, -2.2095]])
'''
# 结论：nn.LogSoftmax = torch.log(nn.Softmax)

loss = nn.NLLLoss()
target1 = torch.tensor([0, 1, 2])  # 随便写一个目标tensor
print(loss(data, target1))  # NLLLoss原始损失
'''
程序运行的一次结果
tensor(0.0157)
(0.2733+0.9425-1.1688)/3 ≈ 0.0157
'''
print(loss(slm(data), target1))  # NLLLoss对LogSoftmax处理后的数据的损失
'''
程序运行的一次结果
tensor(1.5741) 
(1.5084+1.0044+2.2095)/3=1.5741
'''
loss2 = nn.CrossEntropyLoss()
print(loss2(data, target1))  # CrossEntropyLoss损失
'''
程序运行的一次结果
tensor(1.5741)
'''
# 结论：nn.CrossEntropyLoss(input, target1) = nn.NLLLoss(nn.LogSoftmax(input), target1)

target2 = torch.tensor([[1, 0, 0], [0, 1, 0], [0, 0, 1]])  # one-hot 标签
custom_loss = -torch.sum(slm(data) * target2) / 3
print(custom_loss)
'''
程序运行的一次结果
tensor(1.5741)
'''

下面是nn.NLLLoss()函数的公式。
ℓ ( x , y ) = L = l 1 , … , l N ⊤ l n = − w y n x n , y n , w c = w e i g h t [ c ] ⋅ 1 { c ≠ i g n o r e _ i n d e x } ℓ(x,y)=L={l_1,…,l_N}^⊤ \\ l_n=−w_{yn}x_{n,y_n}, w_c=weight[c]⋅1\{c\neq ignore\_index\} \\ ℓ(x,y)=L=l1​,…,lN​⊤ln​=−wyn​xn,yn​​,wc​=weight[c]⋅1{c​=ignore_index}
$$\ell (x,y)=\{\begin{array}{l}{\sum }_{n=1}^N\frac{l_n}{{\sum }_{n=1}^N{w}_{yn}},ifreduction{=}^{\prime }mea{n}^{\prime };\\ {\sum }_{n=1}^N{l}_n,ifreduction{=}^{\prime }su{m}^{\prime }.\end{array}$$
1.默认情况下weight为1，上述代码中，如果NLLLoss有weight参数，那么weight=torch.Tensor([1, 1, 1]),即上述代码中nn.NLLLoss(weight=torch.Tensor([1, 1, 1]))与nn.NLLLoss()等价。
2.默认情况下，nn.NLLLoss()的reduction为mean。