Pytorch: AlexNet在不同的layer使用不同的learning rate

对于一个network:
class AlexNet(nn.Module):

def __init__(self, num_classes=1000):
    super(AlexNet, self).__init__()
    self.features = nn.Sequential(
        nn.Conv2d(3, 64, kernel_size=11, stride=4, padding=2),
        nn.ReLU(inplace=True),
        nn.MaxPool2d(kernel_size=3, stride=2),
        nn.Conv2d(64, 192, kernel_size=5, padding=2),
        nn.ReLU(inplace=True),
        nn.MaxPool2d(kernel_size=3, stride=2),
        nn.Conv2d(192, 384, kernel_size=3, padding=1),
        nn.ReLU(inplace=True),
        nn.Conv2d(384, 256, kernel_size=3, padding=1),
        nn.ReLU(inplace=True),
        nn.Conv2d(256, 256, kernel_size=3, padding=1),
        nn.ReLU(inplace=True),
        nn.MaxPool2d(kernel_size=3, stride=2),
    )
    self.classifier = nn.Sequential(
        nn.Dropout(),
        nn.Linear(256 * 6 * 6, 4096),
        nn.ReLU(inplace=True),
        nn.Dropout(),
        nn.Linear(4096, 4096),
        nn.ReLU(inplace=True),
        nn.Linear(4096, num_classes),
    )

def forward(self, x):
    x = self.features(x)
    x = x.view(x.size(0), 256 * 6 * 6)
    x = self.classifier(x)
    x = F.softmax(x, 1)
    return x

---

对于一个network：

network = AlexNet()
fc3_params = list(map(id, network.classifier[6].parameters()))
base_params = filter(lambda p: id(p) not in fc3_params, network.parameters())
params = [
    {'params': base_params, 'lr': 0.001},
    {'params': network.classifier[6].parameters(), 'lr': 0.01},
]
optimizer = torch.optim.SGD(params, lr=0.001, momentum=0.9, weight_decay=0.0005,)

＃更多层：

network = AlexNet()
fc2_params = list(map(id, network.classifier[4].parameters()))
fc3_params = list(map(id, network.classifier[6].parameters()))
base_params = filter(lambda p: id(p) not in fc3_params+fc2_params, network.parameters())
params = [
    {'params': base_params, 'lr': 0.001},
    {'params': network.classifier[4].parameters(), 'lr': 0.01},
    {'params': network.classifier[6].parameters(), 'lr': 0.1},
]
optimizer = torch.optim.SGD(params, lr=0.001, momentum=0.9, weight_decay=0.0005,)

---