参考链接: class torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0, T_mult=1, eta_min=0, last_epoch=-1, verbose=False)
配套代码下载链接: 测试学习率调度器.zip
实验代码展示1:
# torch.optim.lr_scheduler.CosineAnnealingWarmRestarts
import matplotlib.pyplot as plt
import numpy as np
import torch
from torch.utils.data import Dataset, DataLoader
from torch import nn
from torch.autograd import Function
import random
import os
seed = 20200910
os.environ['PYTHONHASHSEED'] = str(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed(seed)
torch.cuda.manual_seed_all(seed) # if you are using multi-GPU.
np.random.seed(seed) # Numpy module.
random.seed(seed) # Python random module.
torch.manual_seed(seed)
torch.backends.cudnn.benchmark = False
torch.backends.cudnn.deterministic = True
class Dataset4cxq(Dataset):
def __init__(self, length):
self.length = length
def __len__(self):
return self.length
def __getitem__(self, index):
if type(index) != type(2) and type(index) != (slice):
raise TypeError('索引类型错误,程序退出...')
# index 是单个数
if type(index) == type(2):
if index >= self.length or index < -1 * self.length:
# print("索引越界,程序退出...")
raise IndexError("索引越界,程序退出...")
elif index < 0:
index = index + self.length
Celsius = torch.randn(1,1,dtype=torch.float).item()
Fahrenheit = 32.0 + 1.8 * Celsius
return Celsius, Fahrenheit
def collate_fn4cxq(batch):
list_c = []
list_f = []
for c, f in batch:
list_c.append(c)
list_f.append(f)
list_c = torch.tensor(list_c)
list_f = torch.tensor(list_f)
return list_c, list_f
my_dataset_val = Dataset4cxq(16)
dataloader4cxq_val = torch.utils.data.DataLoader(
dataset=my_dataset_val,
batch_size=8,
# batch_size=2,
drop_last=True,
# drop_last=False,
shuffle=True, # True False
# shuffle=False, # True False
collate_fn=collate_fn4cxq,
# collate_fn=None,
)
def validate():
total_loss_val = 0.0
for cnt, data in enumerate(dataloader4cxq_val, 0):
Celsius, Fahrenheit = data
Celsius, Fahrenheit = Celsius.cuda().view(-1,1), Fahrenheit.cuda().view(-1,1)
output = model(Celsius)
loss = cost_function(output, Fahrenheit)
total_loss_val += loss.item()
return total_loss_val
if __name__ == "__main__":
my_dataset = Dataset4cxq(32)
# for c,f in my_dataset:
# print(type(c),type(f))
dataloader4cxq = torch.utils.data.DataLoader(
dataset=my_dataset,
batch_size=8,
# batch_size=2,
drop_last=True,
# drop_last=False,
shuffle=True, # True False
# shuffle=False, # True False
collate_fn=collate_fn4cxq,
# collate_fn=None,
)
# for cnt, data in enumerate(dataloader4cxq, 0):
# # pass
# sample4cxq, label4cxq = data
# print('sample4cxq的类型: ',type(sample4cxq),'\tlabel4cxq的类型: ',type(label4cxq))
# print('迭代次数:', cnt, ' sample4cxq:', sample4cxq, ' label4cxq:', label4cxq)
print('开始创建模型'.center(80,'-'))
model = torch.nn.Linear(in_features=1, out_features=1, bias=True) # True # False
model.cuda()
optimizer = torch.optim.Adam(model.parameters(), lr=0.1)
# optimizer = torch.optim.Adam(model.parameters(), lr=0.01185)
# optimizer = torch.optim.SGD(model.parameters(), lr=0.001, momentum=0.999)
# optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9)
# optimizer = torch.optim.SGD(model.parameters(), lr=0.0001, momentum=0.9999)
# 模拟华氏度与摄氏度之间的转换
# Fahrenheit = 32 + 1.8 * Celsius
model.train()
cost_function = torch.nn.MSELoss()
epochs = 100001 # 100001
epochs = 501 # 100001
print('\n')
print('开始训练模型'.center(80,'-'))
list4delta = list()
list4epoch = list()
scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=10, T_mult=1)
# scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=(lambda epoch: 0.99 ** (epoch//1000)))
# scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr=0.001, steps_per_epoch=len(dataloader4cxq), epochs=epochs)
iters = len(dataloader4cxq)
for epoch in range(epochs):
# with torch.no_grad():
# Celsius = torch.randn(10,1,dtype=torch.float).cuda()
# Fahrenheit = 32.0 + 1.8 * Celsius
# Fahrenheit = Fahrenheit.cuda()
# Celsius = torch.randn(1,1,dtype=torch.float,requires_grad=False).cuda() # requires_grad=False True
# Fahrenheit = 32.0 + 1.8 * Celsius
# Fahrenheit = Fahrenheit.cuda() # requires_grad=False
total_loss = 0.0
for cnt, data in enumerate(dataloader4cxq, 0):
Celsius, Fahrenheit = data
Celsius, Fahrenheit = Celsius.cuda().view(-1,1), Fahrenheit.cuda().view(-1,1)
output = model(Celsius)
loss = cost_function(output, Fahrenheit)
total_loss += loss.item()
optimizer.zero_grad()
loss.backward()
optimizer.step()
scheduler.step(epoch + cnt / iters)
if epoch % 10 == 0: # if epoch % 1000 == 0:
list4delta.append(total_loss)
list4epoch.append(epoch)
if epoch % 50 == 0:
info = '\nepoch:{0:>6}/{1:<6}\t'.format(epoch,epochs)
for k, v in model.state_dict().items():
info += str(k)+ ':' + '{0:<.18f}'.format(v.item()) + '\t'
# info += str(k)+ ':' + str(v.item()) + '\t'
print(info)
fig, ax = plt.subplots()
# ax.plot(10*np.random.randn(100),10*np.random.randn(100),'o')
ax.plot(list4epoch, list4delta, 'r.-', markersize=8)
ax.set_title("Visualization For My Model's Errors")
plt.show()
控制台输出结果展示1:
Windows PowerShell
版权所有 (C) Microsoft Corporation。保留所有权利。
尝试新的跨平台 PowerShell https://aka.ms/pscore6
加载个人及系统配置文件用了 814 毫秒。
(base) PS C:\Users\chenxuqi\Desktop\News4cxq\测试学习率调度器> conda activate pytorch_1.7.1_cu102
(pytorch_1.7.1_cu102) PS C:\Users\chenxuqi\Desktop\News4cxq\测试学习率调度器> & 'D:\Anaconda3\envs\pytorch_1.7.1_cu102\python.exe' 'c:\Users\chenxuqi\.vscode\extensions\ms-python.python-2021.1.502429796\pythonFiles\lib\python\debugpy\launcher' '53361' '--' 'c:\Users\chenxuqi\Desktop\News4cxq\测试学习率调度器\test15.py'
-------------------------------------开始创建模型-------------------------------------
-------------------------------------开始训练模型-------------------------------------
epoch: 0/501 weight:0.868572115898132324 bias:1.375303149223327637
epoch: 50/501 weight:1.061319470405578613 bias:10.887883186340332031
epoch: 100/501 weight:1.525067210197448730 bias:18.510334014892578125
epoch: 150/501 weight:1.753422856330871582 bias:24.145959854125976562
epoch: 200/501 weight:1.755087494850158691 bias:27.898477554321289062
epoch: 250/501 weight:1.804174780845642090 bias:30.107219696044921875
epoch: 300/501 weight:1.831251263618469238 bias:31.238134384155273438
epoch: 350/501 weight:1.802194237709045410 bias:31.735584259033203125
epoch: 400/501 weight:1.802172064781188965 bias:31.921901702880859375
epoch: 450/501 weight:1.801216840744018555 bias:31.980527877807617188
epoch: 500/501 weight:1.800062179565429688 bias:31.996034622192382812
运行结果截图展示1:
实验代码展示2:
# torch.optim.lr_scheduler.CosineAnnealingWarmRestarts
import matplotlib.pyplot as plt
import numpy as np
import torch
from torch.utils.data import Dataset, DataLoader
from torch import nn
from torch.autograd import Function
import random
import os
seed = 20200910
os.environ['PYTHONHASHSEED'] = str(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed(seed)
torch.cuda.manual_seed_all(seed) # if you are using multi-GPU.
np.random.seed(seed) # Numpy module.
random.seed(seed) # Python random module.
torch.manual_seed(seed)
torch.backends.cudnn.benchmark = False
torch.backends.cudnn.deterministic = True
class Dataset4cxq(Dataset):
def __init__(self, length):
self.length = length
def __len__(self):
return self.length
def __getitem__(self, index):
if type(index) != type(2) and type(index) != (slice):
raise TypeError('索引类型错误,程序退出...')
# index 是单个数
if type(index) == type(2):
if index >= self.length or index < -1 * self.length:
# print("索引越界,程序退出...")
raise IndexError("索引越界,程序退出...")
elif index < 0:
index = index + self.length
Celsius = torch.randn(1,1,dtype=torch.float).item()
Fahrenheit = 32.0 + 1.8 * Celsius
return Celsius, Fahrenheit
def collate_fn4cxq(batch):
list_c = []
list_f = []
for c, f in batch:
list_c.append(c)
list_f.append(f)
list_c = torch.tensor(list_c)
list_f = torch.tensor(list_f)
return list_c, list_f
my_dataset_val = Dataset4cxq(16)
dataloader4cxq_val = torch.utils.data.DataLoader(
dataset=my_dataset_val,
batch_size=8,
# batch_size=2,
drop_last=True,
# drop_last=False,
shuffle=True, # True False
# shuffle=False, # True False
collate_fn=collate_fn4cxq,
# collate_fn=None,
)
def validate():
total_loss_val = 0.0
for cnt, data in enumerate(dataloader4cxq_val, 0):
Celsius, Fahrenheit = data
Celsius, Fahrenheit = Celsius.cuda().view(-1,1), Fahrenheit.cuda().view(-1,1)
output = model(Celsius)
loss = cost_function(output, Fahrenheit)
total_loss_val += loss.item()
return total_loss_val
if __name__ == "__main__":
my_dataset = Dataset4cxq(32)
# for c,f in my_dataset:
# print(type(c),type(f))
dataloader4cxq = torch.utils.data.DataLoader(
dataset=my_dataset,
batch_size=8,
# batch_size=2,
drop_last=True,
# drop_last=False,
shuffle=True, # True False
# shuffle=False, # True False
collate_fn=collate_fn4cxq,
# collate_fn=None,
)
# for cnt, data in enumerate(dataloader4cxq, 0):
# # pass
# sample4cxq, label4cxq = data
# print('sample4cxq的类型: ',type(sample4cxq),'\tlabel4cxq的类型: ',type(label4cxq))
# print('迭代次数:', cnt, ' sample4cxq:', sample4cxq, ' label4cxq:', label4cxq)
print('开始创建模型'.center(80,'-'))
model = torch.nn.Linear(in_features=1, out_features=1, bias=True) # True # False
model.cuda()
optimizer = torch.optim.Adam(model.parameters(), lr=0.1)
# optimizer = torch.optim.Adam(model.parameters(), lr=0.01185)
# optimizer = torch.optim.SGD(model.parameters(), lr=0.001, momentum=0.999)
# optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9)
# optimizer = torch.optim.SGD(model.parameters(), lr=0.0001, momentum=0.9999)
# 模拟华氏度与摄氏度之间的转换
# Fahrenheit = 32 + 1.8 * Celsius
model.train()
cost_function = torch.nn.MSELoss()
epochs = 100001 # 100001
epochs = 501 # 100001
print('\n')
print('开始训练模型'.center(80,'-'))
list4delta = list()
list4epoch = list()
scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=10, T_mult=1)
# scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=(lambda epoch: 0.99 ** (epoch//1000)))
# scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr=0.001, steps_per_epoch=len(dataloader4cxq), epochs=epochs)
iters = len(dataloader4cxq)
for epoch in range(epochs):
# with torch.no_grad():
# Celsius = torch.randn(10,1,dtype=torch.float).cuda()
# Fahrenheit = 32.0 + 1.8 * Celsius
# Fahrenheit = Fahrenheit.cuda()
# Celsius = torch.randn(1,1,dtype=torch.float,requires_grad=False).cuda() # requires_grad=False True
# Fahrenheit = 32.0 + 1.8 * Celsius
# Fahrenheit = Fahrenheit.cuda() # requires_grad=False
total_loss = 0.0
for cnt, data in enumerate(dataloader4cxq, 0):
Celsius, Fahrenheit = data
Celsius, Fahrenheit = Celsius.cuda().view(-1,1), Fahrenheit.cuda().view(-1,1)
output = model(Celsius)
loss = cost_function(output, Fahrenheit)
total_loss += loss.item()
optimizer.zero_grad()
loss.backward()
optimizer.step()
scheduler.step()
# scheduler.step(epoch + cnt / iters)
if epoch % 10 == 0: # if epoch % 1000 == 0:
list4delta.append(total_loss)
list4epoch.append(epoch)
if epoch % 50 == 0:
info = '\nepoch:{0:>6}/{1:<6}\t'.format(epoch,epochs)
for k, v in model.state_dict().items():
info += str(k)+ ':' + '{0:<.18f}'.format(v.item()) + '\t'
# info += str(k)+ ':' + str(v.item()) + '\t'
print(info)
fig, ax = plt.subplots()
# ax.plot(10*np.random.randn(100),10*np.random.randn(100),'o')
ax.plot(list4epoch, list4delta, 'r.-', markersize=8)
ax.set_title("Visualization For My Model's Errors")
plt.show()
控制台输出结果展示2:
Windows PowerShell
版权所有 (C) Microsoft Corporation。保留所有权利。
尝试新的跨平台 PowerShell https://aka.ms/pscore6
加载个人及系统配置文件用了 888 毫秒。
(base) PS C:\Users\chenxuqi\Desktop\News4cxq\测试学习率调度器> & 'D:\Anaconda3\envs\pytorch_1.7.1_cu102\python.exe' 'c:\Users\chenxuqi\.vscode\extensions\ms-python.python-2021.1.502429796\pythonFiles\lib\python\debugpy\launcher' '53403' '--' 'c:\Users\chenxuqi\Desktop\News4cxq\测试学习率调度器\test15.py'
-------------------------------------开始创建模型-------------------------------------
-------------------------------------开始训练模型-------------------------------------
epoch: 0/501 weight:0.862339854240417480 bias:1.343463540077209473
epoch: 50/501 weight:1.126684308052062988 bias:11.435602188110351562
epoch: 100/501 weight:1.382040500640869141 bias:19.341432571411132812
epoch: 150/501 weight:1.867864847183227539 bias:24.993761062622070312
epoch: 200/501 weight:1.758726716041564941 bias:28.571763992309570312
epoch: 250/501 weight:1.820813775062561035 bias:30.539505004882812500
epoch: 300/501 weight:1.816300749778747559 bias:31.465360641479492188
epoch: 350/501 weight:1.798230290412902832 bias:31.833845138549804688
epoch: 400/501 weight:1.800919055938720703 bias:31.956798553466796875
epoch: 450/501 weight:1.800506353378295898 bias:31.990676879882812500
epoch: 500/501 weight:1.799997329711914062 bias:31.998388290405273438
运行结果截图展示2:
1万+
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
