在训练神经网络时,"warm-up"策略指的是一种在训练初期逐步增加学习率的方法。
逐步增加学习率:在训练的最初几个epoch中,学习率从一个较小的值开始,然后逐步增加到预设的学习率。
结合其他学习率调度策略:在warm-up阶段之后,通常会切换到另一种学习率调度策略,比如cosine decay、multistep decay等。
SingleSubModel(
(enc): ModuleList(
(0): Sequential(
(0): Sequential(
(0): Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(1): GroupNorm(2, 64, eps=1e-05, affine=True)
(2): ReLU(inplace=True)
)
(1): Sequential(
(0): BasicBlock(
(residual_function): Sequential(
(0): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(1): GroupNorm(2, 64, eps=1e-05, affine=True)
(2): ReLU(inplace=True)
(3): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(4): GroupNorm(2, 64, eps=1e-05, affine=True)
)
(shortcut): Sequential()
)
(1): BasicBlock(
(residual_function): Sequential(
(0): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(1): GroupNorm(2, 64, eps=1e-05, affine=True)
(2): ReLU(inplace=True)
(3): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(4): GroupNorm(2, 64, eps=1e-05, affine=True)
)
(shortcut): Sequential()
)
)
)
(1): Sequential(
(0): BasicBlock(
(residual_function): Sequential(
(0): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
(1): GroupNorm(2, 128, eps=1e-05, affine=True)
(2): ReLU(inplace=True)
(3): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(4): GroupNorm(2, 128, eps=1e-05, affine=True)
)
(shortcut): Sequential(
(0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)
(1): GroupNorm(2, 128, eps=1e-05, affine=True)
)
)
(1): BasicBlock(
(residual_function): Sequential(
(0): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(1): GroupNorm(2, 128, eps=1e-05, affine=True)
(2): ReLU(inplace=True)
(3): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(4): GroupNorm(2, 128, eps=1e-05, affine=True)
)
(shortcut): Sequential()
)
)
(2): Sequential(
(0): BasicBlock(
(residual_function): Sequential(
(0): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
(1): GroupNorm(2, 256, eps=1e-05, affine=True)
(2): ReLU(inplace=True)
(3): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(4): GroupNorm(2, 256, eps=1e-05, affine=True)
)
(shortcut): Sequential(
(0): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)
(1): GroupNorm(2, 256, eps=1e-05, affine=True)
)
)
(1): BasicBlock(
(residual_function): Sequential(
(0): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(1): GroupNorm(2, 256, eps=1e-05, affine=True)
(2): ReLU(inplace=True)
(3): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(4): GroupNorm(2, 256, eps=1e-05, affine=True)
)
(shortcut): Sequential()
)
)
(3): Sequential(
(0): BasicBlock(
(residual_function): Sequential(
(0): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
(1): GroupNorm(2, 512, eps=1e-05, affine=True)
(2): ReLU(inplace=True)
(3): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(4): GroupNorm(2, 512, eps=1e-05, affine=True)
)
(shortcut): Sequential(
(0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)
(1): GroupNorm(2, 512, eps=1e-05, affine=True)
)
)
(1): BasicBlock(
(residual_function): Sequential(
(0): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(1): GroupNorm(2, 512, eps=1e-05, affine=True)
(2): ReLU(inplace=True)
(3): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(4): GroupNorm(2, 512, eps=1e-05, affine=True)
)
(shortcut): Sequential()
)
)
)
(head): Sequential(
(0): AdaptiveAvgPool2d(output_size=(1, 1))
(1): Flatten(start_dim=1, end_dim=-1)
(2): Linear(in_features=512, out_features=100, bias=True)
)
)