深度学习的loss变小梯度是否变小

loss变小梯度反而没有变小。

背景sigmoid(wx+b)的二分类实验，损失函数为平方损失。

实验一:直接训练发现loss减少导致梯度减少。看代码1:

实验二:在wx+b后加上BN层梯loss减少梯度减少。看代码2:

代码1:

from enum import auto
from scipy.io import loadmat
import numpy as np
import torch
import torch.utils.data as data_utils
from torch import nn
import torch.optim as optim
network=nn.Linear(1,1)
tr=torch.Tensor([[-10],[10]])
optimizer = optim.Adam(network.parameters(), lr=0.04)
w=nn.Sigmoid()
i=0
l1=0
while True:
#i=i+1
network.train()
#network1.eval()#
optimizer.zero_grad()
#l=w(network(tr))
l=w(network(tr))
#print(network1(network(tr)))
l=(l[0]-0)**2+(l[1]-1)**2
l.backward()
optimizer.step()
print(l)
if l10:
l1=l
if l1/l>10:
for name, parms in network.named_parameters():
print(‘–>name:’, name)
print(‘–>para:’, parms)
print(‘–>grad_requirs:’,parms.requires_grad)
print(‘–>grad_value:’,parms.grad)
print("=“)
l1=l
代码2:
from enum import auto
from scipy.io import loadmat
import numpy as np
import torch
import torch.utils.data as data_utils
from torch import nn
import torch.optim as optim
network=nn.Linear(1,1)
network1=nn.BatchNorm1d(1)
tr=torch.Tensor([[-10],[10]])
optimizer=optim.Adam(network.parameters(), lr=0.04)
optimizer1=optim.Adam(network1.parameters(), lr=0.04)
w=nn.Sigmoid()
i=0
l1=0
while True:
#i=i+1
network.train()
network1.train()#
optimizer.zero_grad()
optimizer1.zero_grad()
#l=w(network(tr))
l=w(network(tr))
#print(network1(network(tr)))
l=(l[0]-0)**2+(l[1]-1)**2
l.backward()
optimizer.step()
optimizer1.step()
print(l)
if l10:
l1=l
if l1/l>10:
for name, parms in network.named_parameters():
print(‘–>name:’, name)
print(‘–>para:’, parms)
print(‘–>grad_requirs:’,parms.requires_grad)
print(‘–>grad_value:’,parms.grad)
print("=”)
l1=l