【代码学习】神经网络的冻结

在训练神经网络时，"warm-up"策略指的是一种在训练初期逐步增加学习率的方法。
逐步增加学习率：在训练的最初几个epoch中，学习率从一个较小的值开始，然后逐步增加到预设的学习率。
结合其他学习率调度策略：在warm-up阶段之后，通常会切换到另一种学习率调度策略，比如cosine decay、multistep decay等。

SingleSubModel(
  (enc): ModuleList(
    (0): Sequential(
      (0): Sequential(
        (0): Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (1): GroupNorm(2, 64, eps=1e-05, affine=True)
        (2): ReLU(inplace=True)
      )
      (1): Sequential(
        (0): BasicBlock(
          (residual_function): Sequential(
            (0): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
            (1): GroupNorm(2, 64, eps=1e-05, affine=True)
            (2): ReLU(inplace=True)
            (3): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
            (4): GroupNorm(2, 64, eps=1e-05, affine=True)
          )
          (shortcut): Sequential()
        )
        (1): BasicBlock(
          (residual_function): Sequential(
            (0): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
            (1): GroupNorm(2, 64, eps=1e-05, affine=True)
            (2): ReLU(inplace=True)
            (3): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
            (4): GroupNorm(2, 64, eps=1e-05, affine=True)
          )
          (shortcut): Sequential()
        )
      )
    )
    (1): Sequential(
      (0): BasicBlock(
        (residual_function): Sequential(
          (0): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
          (1): GroupNorm(2, 128, eps=1e-05, affine=True)
          (2): ReLU(inplace=True)
          (3): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
          (4): GroupNorm(2, 128, eps=1e-05, affine=True)
        )
        (shortcut): Sequential(
          (0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)
          (1): GroupNorm(2, 128, eps=1e-05, affine=True)
        )
      )
      (1): BasicBlock(
        (residual_function): Sequential(
          (0): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
          (1): GroupNorm(2, 128, eps=1e-05, affine=True)
          (2): ReLU(inplace=True)
          (3): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
          (4): GroupNorm(2, 128, eps=1e-05, affine=True)
        )
        (shortcut): Sequential()
      )
    )
    (2): Sequential(
      (0): BasicBlock(
        (residual_function): Sequential(
          (0): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
          (1): GroupNorm(2, 256, eps=1e-05, affine=True)
          (2): ReLU(inplace=True)
          (3): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
          (4): GroupNorm(2, 256, eps=1e-05, affine=True)
        )
        (shortcut): Sequential(
          (0): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)
          (1): GroupNorm(2, 256, eps=1e-05, affine=True)
        )
      )
      (1): BasicBlock(
        (residual_function): Sequential(
          (0): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
          (1): GroupNorm(2, 256, eps=1e-05, affine=True)
          (2): ReLU(inplace=True)
          (3): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
          (4): GroupNorm(2, 256, eps=1e-05, affine=True)
        )
        (shortcut): Sequential()
      )
    )
    (3): Sequential(
      (0): BasicBlock(
        (residual_function): Sequential(
          (0): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
          (1): GroupNorm(2, 512, eps=1e-05, affine=True)
          (2): ReLU(inplace=True)
          (3): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
          (4): GroupNorm(2, 512, eps=1e-05, affine=True)
        )
        (shortcut): Sequential(
          (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)
          (1): GroupNorm(2, 512, eps=1e-05, affine=True)
        )
      )
      (1): BasicBlock(
        (residual_function): Sequential(
          (0): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
          (1): GroupNorm(2, 512, eps=1e-05, affine=True)
          (2): ReLU(inplace=True)
          (3): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
          (4): GroupNorm(2, 512, eps=1e-05, affine=True)
        )
        (shortcut): Sequential()
      )
    )
  )
  (head): Sequential(
    (0): AdaptiveAvgPool2d(output_size=(1, 1))
    (1): Flatten(start_dim=1, end_dim=-1)
    (2): Linear(in_features=512, out_features=100, bias=True)
  )
)