Finetuning LLMs

微调是预训练之后的步骤，但是也可以使用微调过的模型再进行微调。

• 数据集可以是用于自监督学习的没有标签的数据
• 数据集也可以是有标签的数据
• 数据量比预训练时小的多

这里的微调特指生成式任务上的微调。在这种方式中，

• 需要更新整个模型的权重，而不是像其他模型一样只更新部分权重
• 微调的训练目标与预训练时的目标相同，目的是让模型的输出更加一致
• 有许多先进的方法可以减少对模型的更新

关键点

• 明确的任务是模型是否微调成功的关键（是提炼、扩展、还是什么？）
• 明确意味着清晰定义了模型输出的好和坏的标准

微调步骤：

第一种方法：加载基础模型，加载训练集，训练

1 2 3 4 5

from llama import BasicModelRunner model = BasicModelRunner("EleutherAI/pythia-410m") model.load_data_from_jsonlines("lamini_docs.jsonl", input_key="question", output_key="answer") model.train(is_public=True)

（0）准备数据集

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80

import pandas as pd filename = "lamini_docs.jsonl" instruction_dataset_df = pd.read_json(filename, lines=True) examples = instruction_dataset_df.to_dict() if "question" in examples and "answer" in examples:   text = examples["question"][0] + examples["answer"][0] elif "instruction" in examples and "response" in examples:   text = examples["instruction"][0] + examples["response"][0] elif "input" in examples and "output" in examples:   text = examples["input"][0] + examples["output"][0] else:   text = examples["text"][0] prompt_template = """### Question: {question} ### Answer:""" num_examples = len(examples["question"]) finetuning_dataset = [] for i in range(num_examples):   question = examples["question"][i]   answer = examples["answer"][i]   text_with_prompt_template = prompt_template.format(question=question)   finetuning_dataset.append({"question": text_with_prompt_template, "answer": answer}) from pprint import pprint print("One datapoint in the finetuning dataset:") pprint(finetuning_dataset[0]) def tokenize_function(examples):     if "question" in examples and "answer" in examples:       text = examples["question"][0] + examples["answer"][0]     elif "input" in examples and "output" in examples:       text = examples["input"][0] + examples["output"][0]     else:       text = examples["text"][0]     tokenizer.pad_token = tokenizer.eos_token     tokenized_inputs = tokenizer(         text,         return_tensors="np",         padding=True,     )     max_length = min(         tokenized_inputs["input_ids"].shape[1],         2048     )     tokenizer.truncation_side = "left"     tokenized_inputs = tokenizer(         text,         return_tensors="np",         truncation=True,         max_length=max_length     )     return tokenized_inputs finetuning_dataset_loaded = datasets.load_dataset("json", data_files=filename, split="train") tokenized_dataset = finetuning_dataset_loaded.map(     tokenize_function,     batched=True,     batch_size=1,     drop_last_batch=True ) print(tokenized_dataset) tokenized_dataset = tokenized_dataset.add_column("labels", tokenized_dataset["input_ids"]) split_dataset = tokenized_dataset.train_test_split(test_size=0.1, shuffle=True, seed=123) print(split_dataset) # This is how to push your own dataset to your Huggingface hub !pip install huggingface_hub !huggingface-cli login split_dataset.push_to_hub(dataset_path_hf)

（1）加载数据集

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28

import datasets import tempfile import logging import random import config import os import yaml import logging import time import torch import transformers import pandas as pd from utilities import * from transformers import AutoTokenizer from transformers import AutoModelForCausalLM from transformers import TrainingArguments from transformers import AutoModelForCausalLM from llama import BasicModelRunner from llama import BasicModelRunner logger = logging.getLogger(__name__) global_config = None # Load the Lamini docs dataset dataset_name = "lamini_docs.jsonl" dataset_path = "lamini/lamini_docs" use_hf = True

（2）设置模型，训练配置和tokenizer

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21

# Set up the model, training config, and tokenizer model_name = "EleutherAI/pythia-70m" training_config = {     "model": {         "pretrained_name": model_name,         "max_length" : 2048     },     "datasets": {         "use_hf": use_hf,         "path": dataset_path     },     "verbose": True } tokenizer = AutoTokenizer.from_pretrained(model_name) tokenizer.pad_token = tokenizer.eos_token train_dataset, test_dataset = tokenize_and_split_data(training_config, tokenizer) print(train_dataset) print(test_dataset)

（3）加载基础模型

1 2 3 4 5 6 7 8 9 10 11

# Load the base model base_model = AutoModelForCausalLM.from_pretrained(model_name) device_count = torch.cuda.device_count() if device_count > 0:     logger.debug("Select GPU device")     device = torch.device("cuda") else:     logger.debug("Select CPU device")     device = torch.device("cpu")       base_model.to(device)

（4）定义推理函数

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23

def inference(text, model, tokenizer, max_input_tokens=1000, max_output_tokens=100):     # Tokenize     input_ids = tokenizer.encode(           text,           return_tensors="pt",           truncation=True,           max_length=max_input_tokens     )     # Generate     device = model.device     generated_tokens_with_prompt = model.generate(     input_ids=input_ids.to(device),     max_length=max_output_tokens     )     # Decode     generated_text_with_prompt = tokenizer.batch_decode(generated_tokens_with_prompt, skip_special_tokens=True)     # Strip the prompt     generated_text_answer = generated_text_with_prompt[0][len(text):]     return generated_text_answer

（5）设置训练参数

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56

max_steps = 3 trained_model_name = f"lamini_docs_{max_steps}_steps" output_dir = trained_model_name training_args = TrainingArguments(     # Learning rate     learning_rate=1.0e-5,     # Number of training epochs     num_train_epochs=1,     # Max steps to train for (each step is a batch of data)     # Overrides num_train_epochs, if not -1     max_steps=max_steps,     # Batch size for training     per_device_train_batch_size=1,     # Directory to save model checkpoints     output_dir=output_dir,     # Other arguments     overwrite_output_dir=False, # Overwrite the content of the output directory     disable_tqdm=False, # Disable progress bars     eval_steps=120, # Number of update steps between two evaluations     save_steps=120, # After # steps model is saved     warmup_steps=1, # Number of warmup steps for learning rate scheduler     per_device_eval_batch_size=1, # Batch size for evaluation     evaluation_strategy="steps",     logging_strategy="steps",     logging_steps=1,     optim="adafactor",     gradient_accumulation_steps = 4,     gradient_checkpointing=False,     # Parameters for early stopping     load_best_model_at_end=True,     save_total_limit=1,     metric_for_best_model="eval_loss",     greater_is_better=False ) model_flops = (   base_model.floating_point_ops(     {        "input_ids": torch.zeros(            (1, training_config["model"]["max_length"])       )     }   )   * training_args.gradient_accumulation_steps ) print(base_model) print("Memory footprint", base_model.get_memory_footprint() / 1e9, "GB") print("Flops", model_flops / 1e9, "GFLOPs")

（6）开始训练

1 2 3 4 5 6 7 8 9

trainer = Trainer(     model=base_model,     model_flops=model_flops,     total_steps=max_steps,     args=training_args,     train_dataset=train_dataset,     eval_dataset=test_dataset, ) training_output = trainer.train()

（7）保存模型

1 2 3 4 5 6 7 8

save_dir = f'{output_dir}/final' trainer.save_model(save_dir) print("Saved model to:", save_dir) finetuned_slightly_model = AutoModelForCausalLM.from_pretrained(save_dir, local_files_only=True) finetuned_slightly_model.to(device)

（8）测试

1 2 3 4 5

test_question = test_dataset[0]['question'] print("Question input (test):", test_question) print("Finetuned slightly model's answer: ") print(inference(test_question, finetuned_slightly_model, tokenizer))