pytorch进阶教程

进阶部分

Broadcasting:Expand/without copying data 广播机制

例如：unsqueeze 与 expand，详情见基础部分

import torch
a = torch.randn(4,32,14,14)# 在a上叠加b，改变b的维度使其符合a的标准
b = torch.randn(32,1,1)
#维度从右到左与a对齐
#(4,32,14,14)
#   (32,1,1)
b.unsqueeze(0).expand(4,32,14,14).shape
# torch.Size([4, 32, 14, 14])

为什么要用广播机制？

• 实际需求。比如如下维度：[4,32,8]，代表着[班级，人数，科目分数]，也就是4个班，每个班32个人，每个人有着八个科目的分数，如果我们想为每个同学的每个科目都加五分，越往右维度越细化，[4,32,8]+[5.0] 符合转换需求，就可以自动变换。
• 节省内存
  什么时候需要广播机制？
• 从最后一个维度开始匹配，小维度指定，大维度随意。缺维度就增加一个维度，然后再扩张到正确的维度

import torch
c = torch.tensor([[0,0,0],[10,10,10],[20,20,20],[30,30,30]])
c
'''
tensor([[ 0,  0,  0],
        [10, 10, 10],
        [20, 20, 20],
        [30, 30, 30]])
'''
c2 = c + torch.tensor(5) # 0维，自动扩张
c2
'''
tensor([[ 5,  5,  5],
        [15, 15, 15],
        [25, 25, 25],
        [35, 35, 35]])
'''
c1 = c + torch.tensor([5])# 1维，自动扩张
c1
'''
tensor([[ 5,  5,  5],
        [15, 15, 15],
        [25, 25, 25],
        [35, 35, 35]])
'''
c2 = c + torch.tensor([0,1,2]) # 1维，但size必须为3，才能符合广播机制需求
c2
'''
tensor([[ 0,  1,  2],
        [10, 11, 12],
        [20, 21, 22],
        [30, 31, 32]])
'''
c3= torch.tensor([0,10,20,30]).view(4,1)
c3
'''
tensor([[ 0],
        [10],
        [20],
        [30]])
'''
c4 = torch.tensor([0,1,2])
c5 = c3+c4
c5
'''
tensor([[ 0,  1,  2],
        [10, 11, 12],
        [20, 21, 22],
        [30, 31, 32]])
'''

拼接与拆分

拼接：cat/stack

• cat

a = torch.rand(4,32,8)
b = torch.rand(5,32,8)
#在0维上拼接，必须保证其他维度size一致
torch.cat([a,b],dim=0).shape
# torch.Size([9, 32, 8])
a = torch.rand(4,3,32,32)
b = torch.rand(5,3,32,32)
torch.cat([a,b],dim=0).shape
# torch.Size([9, 3, 32, 32])
a2 = torch.rand(4,1,32,32)
torch.cat([a2,a],dim=1).shape
# torch.Size([4, 4, 32, 32])

• stack: create new dim

# 合并两个班的成绩单
a = torch.rand(32,8)
b = torch.rand(32,8)
torch.stack([a,b],dim=0).shape 
#各增加一个0维，size为1
# torch.Size([2, 32, 8])

拆分：split/chunk

• split

c = torch.rand(2,32,8)
aa,bb=c.split([15,17],dim=1)
aa.shape,bb.shape
# (torch.Size([2, 15, 8]), torch.Size([2, 17, 8]))
aa,bb,cc=c.split([15,14,3],dim=1)
cc.shape
# torch.Size([2, 3, 8])
aa,bb=c.split(16,dim=1) #只有16可以，可以被32整除
aa.shape
# torch.Size([2, 16, 8])

• chunk

aa,bb,cc,dd = c.chunk(4,dim=1)#拆成四份
dd.shape
# torch.Size([2, 8, 8])

基本运算

• 加减乘除

a = torch.rand(3,4)
b =torch.rand(4)
torch.all(torch.eq(a+b,torch.add(a,b)))#加
torch.all(torch.eq(a-b,torch.sub(a,b)))#减
torch.all(torch.eq(a*b,torch.mul(a,b)))#乘
torch.all(torch.eq(a/b,torch.div(a,b)))#除
# tensor(True)

• 2维矩阵相乘

a = torch.full((2,2),3)
b = torch.ones(2,2)
torch.mm(a,b) #only for 2 dim
a@b
'''
tensor([[6., 6.],
        [6., 6.]])
'''

#神经网络的线性层
a = torch.rand(4,784)
x = torch.rand(4,784)
w = torch.rand(512,784) #w惯用写法(channel_out,channel_in)，所以要先将w转置
(x@ w.t()).shape
# torch.Size([4, 512])

#转置小tip
w = torch.rand(512,784,28,28) 
w.transpose(0,3).shape
#torch.Size([28, 784, 28, 512])
w.permute(3,2,0,1).shape
# torch.Size([28, 28, 512, 784])

• 2维以上的矩阵乘法

a = torch.rand(4,3,28,64)
b = torch.rand(4,3,64,32)
torch.matmul(a,b).shape #后面两维做矩阵乘法
# torch.Size([4, 3, 28, 32])

a = torch.rand(4,3,28,64)
b = torch.rand(4,1,64,32)
torch.matmul(a,b).shape 
#既使用了矩阵相乘，又使用了广播机制，b的1维自动扩张
#torch.Size([4, 3, 28, 32])

• 次方运算

a = torch.full([2,2],3)
a.pow(2),aa = a**2
'''
tensor([[9., 9.],
        [9., 9.]])
'''

• 平方根

aa.sqrt() #平方根
'''
tensor([[3., 3.],
        [3., 3.]])
'''
aa.rsqrt() #返回平方根的倒数
'''
tensor([[0.3333, 0.3333],
        [0.3333, 0.3333]])
'''

• e

a  = torch.exp(torch.ones(2,2)) #e^1
'''
tensor([[2.7183, 2.7183],
        [2.7183, 2.7183]])
'''
torch.log(a)#e为底
'''
tensor([[1.0000, 1.0000],
        [1.0000, 1.0000]])
'''

• 近似值

a  = torch.tensor(3.14)
a.floor()#往下取整
a.ceil()#往上取整
a.trunc()#裁剪整数部分
a.frac()#裁剪小数部分
# (tensor(3.), tensor(4.), tensor(3.), tensor(0.1400))
a = torch.tensor(3.46)
a.round()
# tensor(3.)
a = torch.tensor(3.51)
a.round()
# tensor(4.)

• 梯度裁剪clamp

grad = torch.rand(2,3)*15
grad
'''
tensor([[12.9732,  1.2229,  7.0443],
        [14.5265,  5.6721,  2.9501]])
'''
grad.max(axis = 1) # 按行
'''
torch.return_types.max(
values=tensor([12.9732, 14.5265]),-->值 value
indices=tensor([0, 0])) -->索引 index
'''
grad.median()
# tensor(5.6721)

grad.clamp(10)#最小值限定为10(min)
'''
tensor([[12.9732, 10.0000, 10.0000],
        [14.5265, 10.0000, 10.0000]])
'''
grad.clamp(0,10)#限定为 (min,max)
'''
tensor([[10.0000,  1.2229,  7.0443],
        [10.0000,  5.6721,  2.9501]])
'''

统计属性

• norm 范数

a = torch.full([8],1)
b = a.view(2,4)
'''
tensor([[1., 1., 1., 1.],
        [1., 1., 1., 1.]])
'''
c = a.view(2,2,2)
a.norm(1),b.norm(1),c.norm(1) #一范数，∑|x|
# (tensor(8.), tensor(8.), tensor(8.))
a.norm(2),b.norm(2),c.norm(2)#二范数，sqrt(∑|x|^2)
# (tensor(2.8284), tensor(2.8284), tensor(2.8284))
b.norm(1,dim=1)
# tensor([4., 4.])
b.norm(2,dim=1)
# tensor([2., 2.])

a = torch.arange(8).view(2,4).float()
a.min(),a.max(),a.mean(),a.prod()#累乘
# (tensor(0.), tensor(7.), tensor(3.5000), tensor(0.))
a.sum(),a.argmax(),a.argmin()
# (tensor(28.), tensor(7), tensor(0))

• dim/keepdim

a  =torch.randn(4,10)
a.max(dim=1) #获得每一张照片的预测值和置信度 
'''
torch.return_types.max(
values=tensor([0.8487, 1.0158, 2.0120, 1.1418]),#置信度
indices=tensor([1, 4, 8, 3]))#预测值
'''
a.max(dim=1,keepdim=True)#保持二维
'''
torch.return_types.max(
values=tensor([[0.8487],
        [1.0158],
        [2.0120],
        [1.1418]]),
indices=tensor([[1],
        [4],
        [8],
        [3]]))
'''
a.max(dim=1,keepdim=False)
'''
torch.return_types.max(
values=tensor([0.8487, 1.0158, 2.0120, 1.1418]),
indices=tensor([1, 4, 8, 3]))
'''

topk/k-th

a  =torch.randn(4,10)
a.topk(3,dim=1) #1维上最大的三个
'''
torch.return_types.topk(
values=tensor([[0.8487, 0.5053, 0.4132],
        [1.0158, 0.6249, 0.5897],
        [2.0120, 0.7639, 0.5647],
        [1.1418, 0.9470, 0.5445]]),
indices=tensor([[1, 3, 7],
        [4, 1, 7],
        [8, 0, 5],
        [3, 6, 5]]))
'''
a.topk(3,dim=1,largest=False) #最小的三个
'''
torch.return_types.topk(
values=tensor([[-4.0843, -1.1895, -0.7910],
        [-0.9890, -0.9026, -0.8703],
        [-1.4781, -0.8836, -0.7796],
        [-1.6847, -1.4048, -0.7745]]),
indices=tensor([[0, 9, 5],
        [6, 0, 9],
        [1, 6, 2],
        [0, 7, 1]]))
'''
a.kthvalue(8,dim=1) #第八大
'''
torch.return_types.kthvalue(
values=tensor([0.4132, 0.5897, 0.5647, 0.5445]),
indices=tensor([7, 7, 5, 5]))
'''

• compare

a>0, torch.gt(a,0)
#eq() equal()
'''
tensor([[False,  True, False,  True, False, False, False,  True,  True, False],
        [False,  True, False, False,  True, False, False,  True, False, False],
        [ True, False, False, False, False,  True, False, False,  True,  True],
        [False, False, False,  True,  True,  True,  True, False,  True, False]])
'''

高阶Operation

where

torch.where(condition,x,y)–>Tensor

cond  = torch.tensor([[0.6769,0.7271],[0.8884,0.4163]])
cond
'''
tensor([[0.6769, 0.7271],
        [0.8884, 0.4163]])
'''
a = torch.zeros(2,2)
b = torch.ones(2,2)
torch.where(cond>0.5,a,b)#>0.5取a,<0.5取b
'''
tensor([[0., 0.],
        [0., 1.]])
'''

gather

torch.gather(input,dim,index,out=None)–> Tensor
各参数：
索引表：[0,1,2]–>[狗，猫，熊] #input
索引结果：[1,0,1,2] #index
输出结果：[猫，狗，猫，熊] #output

prob = torch.randn(4,10)

idx = prob.topk(dim=1,k=3) #每行前三
idx[1] #topk返回的是值和索引，所以是idx[1]
'''
tensor([[4, 1, 9],
        [8, 3, 9],
        [9, 1, 4],
        [6, 8, 2]])
'''

label = torch.arange(10)+100
label.expand(4,10)
'''
tensor([[100, 101, 102, 103, 104, 105, 106, 107, 108, 109],
        [100, 101, 102, 103, 104, 105, 106, 107, 108, 109],
        [100, 101, 102, 103, 104, 105, 106, 107, 108, 109],
        [100, 101, 102, 103, 104, 105, 106, 107, 108, 109]])
'''
#0-->100,1-->101,2-->102

torch.gather(label.expand(4,10),dim=1,index=idx[1])
'''
tensor([[104, 101, 109],
        [108, 103, 109],
        [109, 101, 104],
        [106, 108, 102]])
'''