Pytorch自动求梯度详解

自动求梯度

Tensor

import torch
import numpy as np

创建一个Tensor，并设置requires_grad=True

x=torch.ones(2,2,requires_grad=True)
print(x)
print(x.grad_fn)

tensor([[1., 1.],
        [1., 1.]], requires_grad=True)
None

y=x+2
print(y)
print(y.grad_fn)

tensor([[3., 3.],
        [3., 3.]], grad_fn=<AddBackward0>)
<AddBackward0 object at 0x000002B8D5235E88>

print(x.is_leaf,y.is_leaf)

True False

z=y*y*3
out=z.mean()
print(z,out)

tensor([[27., 27.],
        [27., 27.]], grad_fn=<MulBackward0>) tensor(27., grad_fn=<MeanBackward0>)

通过.requires_grad_()来用in-place的方式改变requires_grad属性：

a=torch.randn(2,2)# 缺失情况下默认 requires_grad=False
a=((a*3)/(a-1))
print(a.requires_grad)

a.requires_grad_(True)
print(a.requires_grad)

b=(a*a).sum()
print(b.grad_fn)

False
True
<SumBackward0 object at 0x000002B8E09AC188>

梯度

out.backward() #等价于out.backward(torch.tensor(1.))

None

out关于x的梯度

print(x.grad)

tensor([[13.5000, 13.5000],
        [13.5000, 13.5000]])

梯度会累加，因此每次反向传播前应该将梯度清零

out2=x.sum()
out2.backward()
print(x.grad)

out3=x.sum()
x.grad.data.zero_()
out3.backward()
print(x.grad)

tensor([[15.5000, 15.5000],
        [15.5000, 15.5000]])
tensor([[1., 1.],
        [1., 1.]])

示例：

x=torch.tensor([1.0,2.0,3.0,4.0],requires_grad=True)
y=2*x
z=y.view(2,2)
print(z)

tensor([[2., 4.],
        [6., 8.]], grad_fn=<ViewBackward>)

v=torch.tensor([[1.0,0.1],[0.01,0.001]],dtype=torch.float)
z.backward(v)
print(x.grad)

tensor([2.0000, 0.2000, 0.0200, 0.0020])

中断梯度追踪的例子

x=torch.tensor(1.0,requires_grad=True)
y1=x**2
with torch.no_grad(): # y2被no_grad包裹，y2梯度不会回传
    y2=x**2
y3=y1+y2

print(x.requires_grad)
print(y1.requires_grad)
print(y2.requires_grad)
print(y3.requires_grad)

True
True
False
True

y3.backward()
print(x.grad)

tensor(2.)

x=torch.ones(1,requires_grad=True)

print(x.data)
print(x.data.requires_grad)

y=2*x
x.data*=100

y.backward()
print(x)
print(x.grad)

tensor([1.])
False
tensor([100.], requires_grad=True)
tensor([2.])