LLM-微调-方案（一）：Lora【案例：chatGLM-Lora】【在chatGLM原有结构中间插入新的网络层】【微调时冻结原有结构参数，只微调新加入的网络层参数】

Lora主要在模型中注入可训练模块，大模型在预训练完收敛之后模型包含许多进行矩阵乘法的稠密层，这些层通常是满秩的，在微调过程中其实改变量是比较小的，在矩阵乘法中表现为低秩的改变，注入可训练层的目的是想下游微调的低秩改变由可训练层来学习，冻结模型其他部分，大大减少模型训练参数。

这种方法有点类似于矩阵分解，可训练层维度和预训练模型层维度一致为d，先将维度d通过全连接层降维至r，再从r通过全连接层映射回d维度，r<<d，r是矩阵的秩，这样矩阵计算就从d x d变为d x r + r x d，参数量减少很多，上图对矩阵A使用随机高斯初始化，对矩阵B使用0进行初始化。

推理计算的时候，因为没有改变预训练权重，所以换不同的下游任务时，lora模型保存的权重也是可以相应加载进来的，通过矩阵分解的方法参数量减少了很多，且推理时可以并行，对于推理性能并没有增加多少负担，算是比较好的低资源微调方法。

Lora方法包实现：

GitHub - huggingface/peft: PEFT: State-of-the-art Parameter-Efficient Fine-Tuning

p-tuning-v2和Lora两者对于低资源微调大模型的共同点都是冻结大模型参数，通过小模块来学习微调产生的低秩改变。但目前存在的一些问题就是这两种训练方式很容易参数灾难性遗忘，因为模型在微调的时候整个模型层参数未改变，而少参数的学习模块微调时却是改变量巨大，容易给模型在推理时产生较大偏置，使得以前的回答能力被可学习模块带偏，在微调的时候也必须注意可学习模块不能过于拟合微调数据，否则会丧失原本的预训练知识能力，产生灾难性遗忘。

最好能够在微调语料中也加入通用学习语料一起微调，避免产生对微调语料极大的偏向，在instruct gpt论文中也提到在强化学习ppo的时候模型也会很容易对于ppo数据拟合，降低模型通用自然语言任务能力，所以在ppo loss中加入了SFT梯度和预训练梯度来缓解这种遗忘问题。

from transformers.integrations import TensorBoardCallback
from torch.utils.tensorboard import SummaryWriter
from transformers import TrainingArguments
from transformers import Trainer, HfArgumentParser
from transformers import AutoTokenizer, AutoModel
import torch
import torch.nn as nn
from peft import get_peft_model, LoraConfig, TaskType
from dataclasses import dataclass, field
import datasets
import os


tokenizer = AutoTokenizer.from_pretrained("/data/pretrained_models/chatglm-6b", trust_remote_code=True)


@dataclass
class FinetuneArguments:
    dataset_path: str = field(default="data/alpaca")
    model_path: str = field(default="output")
    lora_rank: int = field(default=8)


class CastOutputToFloat(nn.Sequential):
    def forward(self, x):
        return super().forward(x).to(torch.float32)


def data_collator(features: list) -> dict:
    len_ids = [len(feature["input_ids"]) for feature in features]
    longest = max(len_ids)
    input_ids = []
    labels_list = []
    for ids_l, feature in sorted(zip(len_ids, features), key=lambda x: -x[0]):
        ids = feature["input_ids"]  # [37010, 12, 3461, 100, 294, 102, 104, 3539, 2549, 101, 104, 306, 101, 433, ...]
        seq_len = feature["seq_len"]
        labels = ([-100] * (seq_len - 1) + ids[(seq_len - 1) :] + [-100] * (longest - ids_l))
        ids = ids + [tokenizer.pad_token_id] * (longest - ids_l)    # [37010, 12, 3461, 100, 294, 102, 104, 3539, 2549, 101, 104, 306, 101, 433, ...]
        _ids = torch.LongTensor(ids)
        labels_list.append(torch.LongTensor(labels))
        input_ids.append(_ids)
    input_ids = torch.stack(input_ids)  # torch.Size([6, 118])
    labels = torch.stack(labels_list)   # torch.Size([6, 118])

    return {"input_ids": input_ids, "labels": labels}



class ModifiedTrainer(Trainer):
    def compute_loss(self, model, inputs, return_outputs=False):
        loss = model(input_ids=inputs["input_ids"], labels=inputs["labels"]).loss

        return loss

    def save_model(self, output_dir=None, _internal_call=False):
        from transformers.trainer import TRAINING_ARGS_NAME

        os.makedirs(output_dir, exist_ok=True)
        torch.save(self.args, os.path.join(output_dir, TRAINING_ARGS_NAME))
        saved_params = {k: v.to("cpu") for k, v in self.model.named_parameters() if v.requires_grad}
        torch.save(saved_params, os.path.join(output_dir, "adapter_model.bin"))


def main():
    writer = SummaryWriter()
    finetune_args, training_args = HfArgumentParser((FinetuneArguments, TrainingArguments)).parse_args_into_dataclasses()

    # init model
    model = AutoModel.from_pretrained("/data/pretrained_models/chatglm-6b", load_in_8bit=False, trust_remote_code=True, device_map="auto")
    model.gradient_checkpointing_enable()
    model.enable_input_require_grads()
    model.is_parallelizable = True
    model.model_parallel = True
    model.lm_head = CastOutputToFloat(model.lm_head)
    model.config.use_cache = (False)    # silence the warnings. Please re-enable for inference!

    print("原始模型结构：\n", model)
    print("-" * 100)
    for param_tuple in model.named_parameters():
        name, param = param_tuple
        print("layer name = ", name)
        print("-" * 100)

    # setup peft
    peft_config = LoraConfig(task_type=TaskType.CAUSAL_LM, inference_mode=False, r=finetune_args.lora_rank, lora_alpha=32, lora_dropout=0.1)
    model = get_peft_model(model, peft_config)

    print("=" * 200)
    print("添加Lora之后的模型结构：\n", model)
    print("-" * 100)
    for param_tuple in model.named_parameters():
        name, param = param_tuple
        print("layer name = ", name)
        print("-" * 100)

    # load dataset
    dataset = datasets.load_from_disk(finetune_args.dataset_path)   # 'data/alpaca'
    print(f"\n{len(dataset)=}\n")

    # start train
    trainer = ModifiedTrainer(model=model, train_dataset=dataset, args=training_args, callbacks=[TensorBoardCallback(writer)], data_collator=data_collator)

    trainer.train()
    writer.close()
    # save model
    model.save_pretrained(training_args.output_dir)


if __name__ == "__main__":
    main()

ChatGLMForConditionalGeneration(
  (transformer): ChatGLMModel(
    (word_embeddings): Embedding(130528, 4096)
    (layers): ModuleList(
      (0-27): 28 x GLMBlock(
        (input_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)
        (attention): SelfAttention(
          (rotary_emb): RotaryEmbedding()
          (query_key_value): Linear(in_features=4096, out_features=12288, bias=True)
          (dense): Linear(in_features=4096, out_features=4096, bias=True)
        )
        (post_attention_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)
        (mlp): GLU(
          (dense_h_to_4h): Linear(in_features=4096, out_features=16384, bias=True)
          (dense_4h_to_h): Linear(in_features=16384, out_features=4096, bias=True)
        )
      )
    )
    (final_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)
  )
  (lm_head): CastOutputToFloat(
    (0): Linear(in_features=4096, out_features=130528, bias=False)
  )
)

layer name =  transformer.word_embeddings.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.0.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.0.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.0.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.0.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.0.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.0.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.0.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.0.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.0.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.0.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.0.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.0.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.1.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.1.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.1.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.1.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.1.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.1.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.1.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.1.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.1.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.1.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.1.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.1.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.2.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.2.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.2.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.2.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.2.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.2.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.2.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.2.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.2.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.2.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.2.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.2.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.3.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.3.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.3.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.3.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.3.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.3.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.3.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.3.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.3.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.3.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.3.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.3.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.4.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.4.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.4.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.4.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.4.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.4.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.4.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.4.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.4.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.4.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.4.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.4.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.5.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.5.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.5.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.5.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.5.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.5.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.5.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.5.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.5.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.5.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.5.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.5.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.6.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.6.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.6.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.6.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.6.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.6.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.6.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.6.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.6.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.6.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.6.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.6.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.7.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.7.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.7.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.7.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.7.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.7.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.7.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.7.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.7.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.7.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.7.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.7.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.8.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.8.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.8.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.8.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.8.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.8.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.8.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.8.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.8.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.8.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.8.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.8.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.9.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.9.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.9.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.9.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.9.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.9.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.9.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.9.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.9.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.9.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.9.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.9.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.10.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.10.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.10.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.10.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.10.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.10.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.10.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.10.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.10.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.10.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.10.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.10.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.11.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.11.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.11.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.11.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.11.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.11.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.11.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.11.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.11.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.11.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.11.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.11.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.12.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.12.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.12.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.12.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.12.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.12.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.12.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.12.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.12.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.12.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.12.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.12.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.13.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.13.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.13.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.13.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.13.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.13.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.13.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.13.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.13.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.13.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.13.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.13.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.14.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.14.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.14.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.14.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.14.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.14.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.14.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.14.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.14.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.14.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.14.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.14.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.15.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.15.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.15.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.15.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.15.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.15.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.15.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.15.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.15.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.15.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.15.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.15.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.16.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.16.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.16.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.16.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.16.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.16.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.16.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.16.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.16.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.16.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.16.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.16.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.17.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.17.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.17.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.17.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.17.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.17.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.17.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.17.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.17.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.17.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.17.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.17.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.18.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.18.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.18.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.18.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.18.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.18.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.18.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.18.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.18.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.18.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.18.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.18.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.19.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.19.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.19.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.19.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.19.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.19.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.19.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.19.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.19.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.19.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.19.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.19.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.20.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.20.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.20.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.20.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.20.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.20.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.20.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.20.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.20.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.20.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.20.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.20.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.21.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.21.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.21.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.21.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.21.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.21.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.21.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.21.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.21.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.21.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.21.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.21.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.22.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.22.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.22.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.22.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.22.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.22.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.22.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.22.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.22.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.22.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.22.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.22.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.23.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.23.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.23.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.23.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.23.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.23.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.23.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.23.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.23.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.23.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.23.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.23.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.24.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.24.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.24.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.24.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.24.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.24.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.24.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.24.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.24.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.24.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.24.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.24.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.25.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.25.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.25.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.25.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.25.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.25.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.25.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.25.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.25.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.25.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.25.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.25.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.26.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.26.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.26.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.26.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.26.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.26.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.26.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.26.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.26.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.26.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.26.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.26.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.27.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.27.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.27.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.27.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.27.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.27.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.27.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.27.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.27.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.27.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.27.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.layers.27.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  transformer.final_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  transformer.final_layernorm.bias
----------------------------------------------------------------------------------------------------

添加Lora层之后：

PeftModelForCausalLM(
  (base_model): LoraModel(
    (model): ChatGLMForConditionalGeneration(
      (transformer): ChatGLMModel(
        (word_embeddings): Embedding(130528, 4096)
        (layers): ModuleList(
          (0-27): 28 x GLMBlock(
            (input_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)
            (attention): SelfAttention(
              (rotary_emb): RotaryEmbedding()
              (query_key_value): Linear(
                in_features=4096, out_features=12288, bias=True
                


                (lora_dropout): ModuleDict(
                  (default): Dropout(p=0.1, inplace=False)
                )
                (lora_A): ModuleDict(
                  (default): Linear(in_features=4096, out_features=8, bias=False)
                )
                (lora_B): ModuleDict(
                  (default): Linear(in_features=8, out_features=12288, bias=False)
                )
                (lora_embedding_A): ParameterDict()
                (lora_embedding_B): ParameterDict()



              )
              (dense): Linear(in_features=4096, out_features=4096, bias=True)
            )
            (post_attention_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)
            (mlp): GLU(
              (dense_h_to_4h): Linear(in_features=4096, out_features=16384, bias=True)
              (dense_4h_to_h): Linear(in_features=16384, out_features=4096, bias=True)
            )
          )
        )
        (final_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)
      )
      (lm_head): CastOutputToFloat(
        (0): Linear(in_features=4096, out_features=130528, bias=False)
      )
    )
  )
)

layer name =  base_model.model.transformer.word_embeddings.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.0.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.0.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.0.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.0.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.0.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.0.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.0.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.0.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.0.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.0.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.0.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.0.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.0.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.0.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.1.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.1.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.1.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.1.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.1.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.1.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.1.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.1.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.1.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.1.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.1.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.1.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.1.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.1.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.2.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.2.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.2.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.2.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.2.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.2.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.2.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.2.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.2.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.2.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.2.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.2.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.2.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.2.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.3.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.3.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.3.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.3.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.3.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.3.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.3.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.3.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.3.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.3.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.3.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.3.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.3.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.3.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.4.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.4.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.4.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.4.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.4.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.4.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.4.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.4.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.4.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.4.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.4.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.4.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.4.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.4.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.5.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.5.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.5.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.5.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.5.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.5.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.5.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.5.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.5.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.5.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.5.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.5.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.5.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.5.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.6.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.6.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.6.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.6.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.6.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.6.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.6.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.6.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.6.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.6.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.6.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.6.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.6.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.6.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.7.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.7.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.7.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.7.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.7.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.7.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.7.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.7.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.7.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.7.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.7.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.7.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.7.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.7.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.8.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.8.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.8.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.8.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.8.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.8.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.8.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.8.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.8.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.8.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.8.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.8.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.8.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.8.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.9.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.9.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.9.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.9.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.9.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.9.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.9.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.9.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.9.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.9.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.9.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.9.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.9.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.9.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.10.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.10.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.10.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.10.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.10.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.10.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.10.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.10.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.10.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.10.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.10.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.10.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.10.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.10.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.11.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.11.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.11.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.11.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.11.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.11.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.11.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.11.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.11.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.11.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.11.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.11.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.11.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.11.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.12.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.12.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.12.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.12.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.12.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.12.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.12.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.12.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.12.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.12.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.12.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.12.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.12.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.12.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.13.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.13.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.13.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.13.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.13.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.13.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.13.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.13.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.13.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.13.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.13.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.13.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.13.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.13.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.14.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.14.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.14.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.14.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.14.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.14.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.14.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.14.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.14.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.14.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.14.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.14.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.14.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.14.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.15.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.15.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.15.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.15.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.15.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.15.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.15.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.15.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.15.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.15.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.15.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.15.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.15.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.15.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.16.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.16.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.16.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.16.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.16.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.16.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.16.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.16.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.16.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.16.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.16.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.16.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.16.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.16.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.17.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.17.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.17.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.17.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.17.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.17.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.17.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.17.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.17.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.17.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.17.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.17.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.17.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.17.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.18.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.18.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.18.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.18.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.18.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.18.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.18.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.18.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.18.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.18.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.18.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.18.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.18.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.18.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.19.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.19.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.19.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.19.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.19.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.19.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.19.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.19.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.19.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.19.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.19.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.19.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.19.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.19.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.20.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.20.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.20.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.20.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.20.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.20.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.20.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.20.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.20.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.20.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.20.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.20.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.20.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.20.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.21.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.21.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.21.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.21.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.21.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.21.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.21.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.21.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.21.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.21.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.21.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.21.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.21.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.21.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.22.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.22.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.22.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.22.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.22.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.22.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.22.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.22.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.22.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.22.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.22.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.22.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.22.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.22.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.23.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.23.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.23.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.23.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.23.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.23.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.23.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.23.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.23.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.23.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.23.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.23.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.23.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.23.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.24.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.24.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.24.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.24.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.24.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.24.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.24.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.24.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.24.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.24.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.24.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.24.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.24.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.24.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.25.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.25.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.25.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.25.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.25.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.25.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.25.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.25.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.25.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.25.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.25.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.25.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.25.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.25.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.26.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.26.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.26.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.26.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.26.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.26.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.26.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.26.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.26.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.26.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.26.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.26.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.26.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.26.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.27.input_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.27.input_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.27.attention.query_key_value.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.27.attention.query_key_value.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.27.attention.query_key_value.lora_A.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.27.attention.query_key_value.lora_B.default.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.27.attention.dense.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.27.attention.dense.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.27.post_attention_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.27.post_attention_layernorm.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.27.mlp.dense_h_to_4h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.27.mlp.dense_h_to_4h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.27.mlp.dense_4h_to_h.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.layers.27.mlp.dense_4h_to_h.bias
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.final_layernorm.weight
----------------------------------------------------------------------------------------------------
layer name =  base_model.model.transformer.final_layernorm.bias
----------------------------------------------------------------------------------------------------

---

---

---

ChatGLM-6B模型微调实战（以 ADGEN (广告生成) 数据集为例，序列长度达 2048）_桂花很香,旭很美的博客-CSDN博客

https://songshanhu.csdn.net/64425c1dae650e245cfead85.html

https://devpress.csdn.net/chuangye/6438f5c2986c660f3cf93bbb.html

LLM大模型低资源微调p tuning v2和lora区别 - 知乎

【自然语言处理】【大模型】极低资源微调大模型方法LoRA以及BLOOM-LORA实现代码 - 知乎