Pytroch大模型Fine-tune实用小技巧之打印每层参数名及参数冻结

对于大模型Fine-tune，必须冻结大部分低层神经网络的参数，以下命令用于查看模型冻结情况，以便调优。

    for name, parameter in model.named_parameters():
        # 如果只想训练bert的参数的话：
        name = str(name).split('.') # 将torch类型转换为string以便处理
        print(name)
        print(type(name)) 
        # 根据层的名称冻结参数
        if len(name)>3:  # 
            if name[3].isalpha() :
                if name[1] == 'pooler' :
                    parameter.requires_grad = True
                else :
                    parameter.requires_grad = False
            else :
                if int(name[3])>5:
                    parameter.requires_grad = True
                else :
                    parameter.requires_grad = False
        else:
            parameter.requires_grad = True
        # print([p.requires_grad for name, p in model.named_parameters()])

输出结果如下：

bert.embeddings.word_embeddings.weight
True
bert.embeddings.position_embeddings.weight
True
bert.embeddings.token_type_embeddings.weight
True
bert.embeddings.LayerNorm.weight
True
bert.embeddings.LayerNorm.bias
True
bert.encoder.layer.0.attention.self.query.weight
True
bert.encoder.layer.0.attention.self.query.bias
True
bert.encoder.layer.0.attention.self.key.weight
True
bert.encoder.layer.0.attention.self.key.bias
True
bert.encoder.layer.0.attention.self.value.weight
True
bert.encoder.layer.0.attention.self.value.bias
True
bert.encoder.layer.0.attention.output.dense.weight
True
bert.encoder.layer.0.attention.output.dense.bias
True
bert.encoder.layer.0.attention.output.LayerNorm.weight
True
bert.encoder.layer.0.attention.output.LayerNorm.bias
True
bert.encoder.layer.0.intermediate.dense.weight
True
bert.encoder.layer.0.intermediate.dense.bias
True
bert.encoder.layer.0.output.dense.weight
True
bert.encoder.layer.0.output.dense.bias
True
bert.encoder.layer.0.output.LayerNorm.weight
True
bert.encoder.layer.0.output.LayerNorm.bias
True
bert.encoder.layer.1.attention.self.query.weight
True
bert.encoder.layer.1.attention.self.query.bias
True
bert.encoder.layer.1.attention.self.key.weight
True
bert.encoder.layer.1.attention.self.key.bias
True
bert.encoder.layer.1.attention.self.value.weight
True
b