基础积累-ML

1. 交叉熵：

2分类交叉熵:

- ylogp + (-1) * (1-y)log(1-p)

p是预测概率，经过sigmoid处理

import torch

def binary_cross_entropy_torch(y_true, y_pred):
    """
    使用PyTorch从0实现二分类交叉熵损失函数

    参数:
    y_true: 真实标签，PyTorch Tensor，形状为(batch_size,)，包含0或1
    y_pred: 预测概率，PyTorch Tensor，形状为(batch_size,)，包含预测为正类的概率

    返回:
    loss: 二分类交叉熵损失值，一个标量
    """
    epsilon = 1e-15  # 防止出现log(0)的情况
    y_pred = torch.clamp(y_pred, epsilon, 1. - epsilon)  # 确保预测概率在合理范围内

    loss = -(y_true * torch.log(y_pred) + (1 - y_true) * torch.log(1 - y_pred))
    loss = torch.mean(loss)  # 计算batch的平均损失
    return loss

# 示例
y_true = torch.tensor([1, 0, 1, 0], dtype=torch.float32)  # 真实标签
y_pred = torch.tensor([0.9, 0.1, 0.8, 0.4], dtype=torch.float32)  # 模型预测的概率

loss = binary_cross_entropy_torch(y_true, y_pred)
print("Binary Cross Entropy Loss:", loss.item())

多分类交叉熵

-y*logp

p是softmax处理之后的结果

其实结果就是-1 * logsoftmax(pred_i), i = label_index

import torch
import torch.nn.functional as F

def cross_entropy_loss(logits, labels):
    """
    logits: [batch_size, num_classes] 未归一化的预测值（logits）
    labels: [batch_size] 真实的类别索引
    """
    # 将logits通过softmax函数转换为概率分布
    probabilities = F.softmax(logits, dim=1)
    
    # 根据真实标签提取每个样本对应类别的预测概率
    probabilities_for_labels = probabilities[range(len(labels)), labels]
    
    # 计算交叉熵损失
    loss = -torch.log(probabilities_for_labels).mean()
    
    return loss

# 示例用法
batch_size = 3
num_classes = 5
logits = torch.randn(batch_size, num_classes)  # 模拟logits
labels = torch.randint(0, num_classes, (batch_size,))  # 模拟真实标签

loss = cross_entropy_loss(logits, labels)
print("Cross Entropy Loss:", loss.item())

NCELoss

InfoNCELoss

SIMCSE loss:

def compute_loss(y_pred, lamda=0.05, add=None):
    idxs = torch.arange(0, y_pred.shape[0])  # [0,1,2,3,4,5]
    y_true = idxs + 1 - idxs % 2 * 2  # 生成真实的label  = [1,0,3,2,5,4]

    y_true = y_true.to("cuda")

    # 对句向量进行L2正则化
    if add is not None:
        y_pred = torch.cat([y_pred, add], dim=0)

    y_pred = torch.nn.functional.normalize(y_pred, p=2, dim=1)

    similarities = F.cosine_similarity(y_pred.unsqueeze(1), y_pred.unsqueeze(0), dim=2)

    similarities = similarities - torch.eye(y_pred.shape[0], device="cuda") * 1e12
    if add is not None:
        similarities = similarities[:y_true.shape[0]]

    # 论文中除以 temperature 超参
    similarities = similarities / lamda
    similarities = similarities.to("cuda")
    loss = F.cross_entropy(similarities, y_true)
    return torch.mean(loss)