PyTorch: 交叉熵

1. 导入包

import torch
from torch.nn import functional as F

2. 熵Entropy

#熵
#uncertainty越高，信息越多；entropy越高，信息越少
#Entropy=-P(i)*log2[P(i)]的求和

print("熵示例一：")
a=torch.full([4],0.25)
print(a)#tensor([0.2500, 0.2500, 0.2500, 0.2500])
print(-(a*torch.log2(a)).sum())#tensor(2.),熵最大，所含的信息量最少

b=torch.tensor([0.1,0.1,0.1,0.7])
print(-(b*torch.log2(b)).sum())#tensor(1.3568)，熵较大，所含的信息量较少

c=torch.tensor([0.001,0.001,0.001,0.997])
print(-(c*torch.log2(c)).sum())#tensor(0.0342)，不确定性大，熵小，所含的信息量最多

3. 交叉熵Cross Entropy

#交叉熵
#当P=Q时，H(p,Q)=H(P)=-P(i)*log2[P(i)].sum()
#当P!=Q时，H(P,Q)=-P(i)*log2[Q(i)].sum()
#二值分类：cat和dog
  #P(dog)=1-P(cat),Q(dog)=1-Q(cat)
  #H(P,Q)=-P(i)*log2[Q(i)].sum()=-P(cat)*log2(cat)-P(dog)*log2(dog)=-[y*log2(x)+(1-y)*log2(1-x)]
#分类问题使用交叉熵：cross entropy,而不使用mse

print("示例二：")
x=torch.randn(1,784)
w=torch.randn(10,784)

logits=x@w.t()
print(logits.shape)

pred=F.softmax(logits,dim=1)#得到概率值
print(pred.shape)
pred_log=torch.log(pred)

#自动得到softmax后的结果，不用进行softmax操作
#cross_entropy=softmax+log+nll_loss
result1=F.cross_entropy(logits,torch.tensor([3]))
print(result1)#tensor(80.8452)
#不包含softmax操作
result2=F.nll_loss(pred_log,torch.tensor([3]))
print(result2)#tensor(80.8452)