python调度器_Python可视化各种LR_u调度器（学习率）,python,lrscheduler

1.torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max, eta_min=0, last_epoch=-1, verbose=False)

实现代码:

import torch

import torch.nn as nn

import itertools

import matplotlib.pyplot as plt

initial_lr = 0.1

epochs = 100

# 定义一个简单的模型

class model(nn.Module):

def __init__(self):

super().__init__()

self.conv1 = nn.Conv2d(in_channels=3, out_channels=3, kernel_size=3)

def forward(self, x):

pass

if __name__ == '__main__':

net = model()

optimizer = torch.optim.Adam(net.parameters(), lr=initial_lr)

scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=5)

print("初始化的学习率：", optimizer.defaults['lr'])

lr_list = [] # 保存学习率

for epoch in range(epochs):

# train

optimizer.zero_grad()

optimizer.step()

# print("第%d个epoch的学习率：%f" % (epoch, optimizer.param_groups[0]['lr']))

lr_list.append(optimizer.param_groups[0]['lr'])

scheduler.step()

# 画出lr的变化

plt.plot(list(range(epochs)), lr_list)

plt.xlabel("epoch")

plt.ylabel("lr")

plt.title("CosineAnnealingLR")

plt.show()

2.CosineAnnealingWarmRestarts

T_mult = 1

T_mult = 2

scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=5, T_mult=1)

3.torch.optim.lr_scheduler.StepLR(optimizer, step_size, gamma=0.1, last_epoch=-1, verbose=False)

scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=30)

4.torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones, gamma=0.1, last_epoch=-1, verbose=False)

scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[30,80])

5.torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma, last_epoch=-1, verbose=False)

scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma = 0.95)

6.torch.optim.lr_scheduler.CyclicLR(optimizer, base_lr, max_lr, step_size_up=2000, step_size_down=None, mode=‘triangular’, gamma=1.0, scale_fn=None, scale_mode=‘cycle’, cycle_momentum=True, base_momentum=0.8, max_momentum=0.9, last_epoch=-1, verbose=False)

mode = ‘triangular’

import torch

import torch.nn as nn

import itertools

import matplotlib.pyplot as plt

from utils.transform import test_preprocess

from torchvision import datasets

initial_lr = 0.1

epochs = 10

# 这里为数据集路径

TEST_PATH = r'D:\home_image\test'

data = datasets.ImageFolder(root=TEST_PATH, transform=test_preprocess)

data_loader = torch.utils.data.DataLoader(data, batch_size=25, num_workers=config.num_workers)

# 定义一个简单的模型

class model(nn.Module):

def __init__(self):

super().__init__()

self.conv1 = nn.Conv2d(in_channels=3, out_channels=3, kernel_size=3)

def forward(self, x):

pass

if __name__ == '__main__':

net = model()

optimizer = torch.optim.SGD(net.parameters(), lr=initial_lr)

scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer, base_lr=0.01, max_lr=0.1,

mode='triangular',

step_size_up=10, cycle_momentum=True)

print("初始化的学习率：", optimizer.defaults['lr'])

lr_list = [] # 保存学习率

for epoch in range(epochs):

for batch in data_loader:

# train

optimizer.zero_grad()

optimizer.step()

# print("第%d个epoch的学习率：%f" % (epoch, optimizer.param_groups[0]['lr']))

lr_list.append(optimizer.param_groups[0]['lr'])

scheduler.step()

# 画出lr的变化

plt.plot(list(range(epochs*len(data_loader))), lr_list)

plt.xlabel("iteration")

plt.ylabel("lr")

plt.title("CyclicLR-exp_range")

plt.show()

mode = ‘triangular2’

scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer, base_lr=0.01, max_lr=0.1,

mode='triangular2',

step_size_up=10, cycle_momentum=True)

mode = ‘exp_range’, gamma = 0.98

scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer, base_lr=0.01, max_lr=0.1,

mode='exp_range', gamma=0.98,

step_size_up=10, cycle_momentum=True)

7.torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr, total_steps=None, epochs=None, steps_per_epoch=None, pct_start=0.3, anneal_strategy=‘cos’, cycle_momentum=True, base_momentum=0.85, max_momentum=0.95, div_factor=25.0, final_div_factor=10000.0, last_epoch=-1, verbose=False)

epochs = 20

scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr=0.01,

steps_per_epoch=len(data_loader), epochs=epochs)