【pytorch学习】nn.CrossEntropyLoss，label_smooth

文章：
简单的label smooth为什么能够涨点？

label smooth标签平滑的理解

---

Pytorch 官网：

# from Pytorch官网：https://pytorch.org/docs/stable/generated/torch.nn.CrossEntropyLoss.html

import torch
import torch.nn as nn

# Example of target with class indices
loss = nn.CrossEntropyLoss(label_smoothing=0.1)
input = torch.randn(3, 5, requires_grad=True)
target = torch.empty(3, dtype=torch.long).random_(5)
output = loss(input, target)
print(output)
# output.backward()
# Example of target with class probabilities
input = torch.randn(3, 5, requires_grad=True)
target = torch.randn(3, 5).softmax(dim=1)
output = loss(input, target)
print(output)
# output.backward()

实现原理：

# from: https://github.com/guochengqian/openpoints/blob/ee100c81b1d9603c0fc76a3ee4e37d10b2af60ba/loss/cross_entropy.py

""" Cross Entropy w/ smoothing or soft targets

Borrowed from Ross Wightman (https://www.github.com/timm)
"""

import torch
import torch.nn as nn
import torch.nn.functional as F
# from .build import LOSS


# @LOSS.register_module()
class LabelSmoothingCrossEntropy(nn.Module):  # Example of target with class probabilities
    """ NLL loss with label smoothing.
    """
    def __init__(self, label_smoothing=0.1):
        super(LabelSmoothingCrossEntropy, self).__init__()
        self.smoothing = label_smoothing
        self.confidence = 1. - self.smoothing

    def forward(self, x: torch.Tensor, target: torch.Tensor) -> torch.Tensor:
        logprobs = F.log_softmax(x, dim=1)
        nll_loss = -logprobs.gather(dim=1, index=target.unsqueeze(1))
        nll_loss = nll_loss.squeeze(1)
        smooth_loss = -logprobs.mean(dim=-1)
        loss = self.confidence * nll_loss + self.smoothing * smooth_loss
        return loss.mean()


# @LOSS.register_module()
class SoftTargetCrossEntropy(nn.Module): # Example of target with class indices

    def __init__(self, **kwargs):
        super(SoftTargetCrossEntropy, self).__init__()

    def forward(self, x: torch.Tensor, target: torch.Tensor) -> torch.Tensor:
        loss = torch.sum(-target * F.log_softmax(x, dim=-1), dim=-1)
        return loss.mean()

验证：

loss = nn.CrossEntropyLoss(label_smoothing=0.1)
input = torch.randn(3, 5, requires_grad=True)
target = torch.arange(3)
target
output = loss(input, target)
print(input)
print(target)
print(output)

tensor([[ 0.1871,  0.4518, -0.3632, -0.7217,  0.0318],
        [ 0.9812, -0.5675, -1.3484, -1.0167,  1.2187],
        [-0.4820, -0.2057, -0.0044,  1.6346, -0.3886]], requires_grad=True)
tensor([0, 1, 2])
tensor(2.0178, grad_fn=<AddBackward0>)
-------------------------------------------
loss = LabelSmoothingCrossEntropy()
output = loss(input, target)
print(output)

tensor(2.0178, grad_fn=<MeanBackward0>)