cosine similarity contrastive loss

TwinBERT: logistic regression layer is applied to the cosine similarity score and convert it to [0, 1].

TwinBERT: Distilling Knowledge to Twin-Structured BERT Models for Efficient Retrieval https://arxiv.org/pdf/2002.06275.pdf

Learning Text Similarity with Siamese Recurrent Networks https://aclanthology.org/W16-1617.pdf

class CosineContrastiveLoss(nn.Module):
    def __init__(self, margin=0.4):
        super(CosineContrastiveLoss, self).__init__()
        self.margin = margin

    def forward(self, output1, output2, label):
        cos_sim = F.cosine_similarity(output1, output2)
        loss_cos_con = torch.mean((1-label) * torch.div(torch.pow((1.0-cos_sim), 2), 4) +
                                    (label) * torch.pow(cos_sim * torch.lt(cos_sim, self.margin), 2))
        return loss_cos_con