【Foundation】（三）transformers之Model

本篇博客内容以及后续内容均来自b站up主你可是处女座啊

1、介绍

1.1、 模型类型

• 编码器模型：自编码器模型，使用encoder，拥有双向注意力机制，即计算每一个词的特征时都看到完整的上下文
• 解码器模型：自回归模型，使用decoder，拥有单向注意力机制，即计算每一个词的特征时智能看到上文，无法看到下文
• 编码解码器模型：序列到序列的模型，使用encoder-decoder encoder使用双向注意力，decoder使用单向注意力
  

1.2、Model Head

2、模型加载

from transformers import  AutoConfig,AutoModel,AutoTokenizer

#在线加载
model = AutoModel.from_pretrained('hfl/rbt3')

#模型下载
#!git clone “https://huggingface.co/hfl/rbt3”
!git lfs clone “https://huggingface.co/hfl/rbt3” --include=“*.bin”

#离线加载
model = AutoModel.from_pretrained('rbt3')

模型参数
model.config

3、模型调用

sen = '弱小的我也有大梦想'
tokenizer = AutoTokenizer.from_pretrained('hfl/rbt3',output_attentions=True)
inputs = tokenizer(sen,return_tensors='pt')
inputs

3.1、不带Model Head的模型调用

model = AutoModel.from_pretrained('hfl/rbt3',output_attentions=True)
output = model(**inputs)
output

output.last_hidden_state.size()

3.2、带Model Head的模型调用

from transformers import AutoModelForSequenceClassification,BertForSequenceClassification
clz_model = AutoModelForSequenceClassification.from_pretrained('hfl/rbt3',num_labels=10)
clz_model(**inputs)

clz_model.config.id2label

clz_model.config.num_labels

4、模型微调实战

4.1、导包

#文本分类实战
import pandas as pd
from datasets import load_dataset
from transformers import AutoTokenizer,AutoModelForSequenceClassification,Trainer,TrainingArguments

4.2、加载数据

#法一
data = pd.read_csv('./datasets/ChnSentiCorp_htl_all.csv')
data.head()

data = data.dropna()
data

#法二
dataset = load_dataset('csv',data_files='datasets/ChnSentiCorp_htl_all.csv',split='train')
dataset = dataset.filter(lambda x : x['review'] is not None)
dataset

4.3、创建数据集

from torch.utils.data import Dataset
class MyDataset(Dataset):
    def __init__(self) -> None:
        super().__init__()
        self.data =pd.read_csv('./datasets/ChnSentiCorp_htl_all.csv')
        self.data = self.data.dropna()

    def __getitem__(self, index):
        return self.data.iloc[index]['review'],self.data.iloc[index]['label']
    def __len__(self):
        return len(self.data)

dataset = MyDataset()
for i in range(5):
    print(dataset[i])

4.4、划分数据集

from torch.utils.data import random_split
#法一
trainset,validset = random_split(dataset,lengths=[0.9,0.1])
len(trainset),len(validset)

#法二
dataset = dataset.train_test_split(test_size=0.1)
dataset

4.5、创建加载器

from transformers import AutoTokenizer
import torch
tokenizer =AutoTokenizer.from_pretrained('hfl/rbt3')
def collate_func(batch):
    texts,labels = [],[]
    for item in batch:
        texts.append(item[0])
        labels.append(item[1])
    inputs = tokenizer(texts,max_length=128,padding='max_length',truncation=True,
                       return_tensors='pt')
    inputs['labels']= torch.tensor(labels)
    return inputs

from torch.utils.data import DataLoader
trainloader = DataLoader(trainset,batch_size=32,shuffle=True,collate_fn=collate_func)
validloader = DataLoader(trainset,batch_size=64,shuffle=False,collate_fn=collate_func)

import torch
tokenizer =AutoTokenizer.from_pretrained('rbt3')


def process_function(examples):
    tokenized_examples = tokenizer(examples['review'],max_length=128,truncation=True)
    tokenized_examples['labels'] = examples["label"]
    return tokenized_examples


tokenized_datasets = dataset.map(process_function,batched=True,remove_columns=dataset['train'].column_names)
tokenized_datasets

from torch.utils.data import DataLoader
from transformers import DataCollatorWithPadding
trainset ,validset = tokenized_datasets['train'],tokenized_datasets['test']

trainloader = DataLoader(trainset,batch_size=32,shuffle=True,collate_fn=DataCollatorWithPadding(tokenizer))
validloader = DataLoader(validset,batch_size=32,shuffle=True,collate_fn=DataCollatorWithPadding(tokenizer))

4.6、创建模型以及优化器

from torch.optim import Adam
from transformers import AutoModelForSequenceClassification
#法一
model = AutoModelForSequenceClassification.from_pretrained('hfl/rbt3')
if torch.cuda.is_available():
   model.to('cuda')
opt = Adam(model.parameters(),lr=2e-5)

#方法二 trainer
model = AutoModelForSequenceClassification.from_pretrained('rbt3')

########## 创建评估函数
import evaluate

acc_metric = evaluate.load("accuracy")
f1_metric = evaluate.load("f1")

def eval_metric(eval_predict):
    predictions,labels = eval_predict
    predictions = predictions.argmax(axis=-1)
    acc = acc_metric.compute(predictions=predictions, references=labels)
    f1 = f1_metric.compute(predictions=predictions, references=labels)
    acc.update(f1)
    return acc

4.7、模型训练

import evaluate
clf_metrics = evaluate.combine(['accuracy', 'f1'])

def train(epoch=3,log_step=100):
    global_step = 0
    for ep in range(epoch):
        model.train()
        for batch in trainloader:
            if torch.cuda.is_available():
                batch = {k:v.to('cuda') for k,v in batch.items()}
            opt.zero_grad()
            outputs = model(**batch)
            loss = outputs.loss
            loss.backward()
            opt.step()
            global_step += 1
            if global_step % log_step == 0:
                print(f'epoch:{ep},global_step:{global_step},loss:{loss.item()}')
        acc = evaluate()
        # print(f'ep:{ep},acc:{acc}')
        print(f'ep:{ep},{acc}')
def evaluate():
    model.eval()
    acc_num = 0
    with torch.inference_mode():
        for batch in validloader:
            if torch.cuda.is_available():
                batch = {k:v.to('cuda') for k,v in batch.items()}
            outputs = model(**batch)
            pred = torch.argmax(outputs.logits,dim=-1)
            clf_metrics.add_batch(predictions=pred.long(),references=batch['labels'].long())
            
    return clf_metrics.compute()
            # acc_num += (pred.long() == batch['labels'].long()).float().sum()
    # return acc_num / len(validset)

#创建Training Arguments

train_args = TrainingArguments(output_dir='./checkpoint',       #输出文件
                               per_device_eval_batch_size=8,    #验证时batch大小
                               per_device_train_batch_size=8,   #训练时batch大小
                               logging_steps=10,                #每10步打印日志
                               eval_strategy="epoch",     #评估策略  epoch、step
                               save_steps=100,                  #每100步保存一次模型
                               save_strategy='epoch',           #保存策略
                               save_total_limit=2,              #保存模型的数量
                               learning_rate=2e-5,              #学习率
                               weight_decay=1e-5,               #衰减率
                               metric_for_best_model="f1",      #最好模型评估标准
                               load_best_model_at_end = True,   #加载最优模型
                               )
train_args

#创建Trainer
from transformers import DataCollatorWithPadding
trainer = Trainer(
    model = model,
    args=train_args,
    train_dataset=tokenized_datasets['train'],
    eval_dataset=tokenized_datasets['test'],
    data_collator=DataCollatorWithPadding(tokenizer=tokenizer),
    compute_metrics=eval_metric
    )

trainer.train()

4.8、模型评估

trainer.evaluate(tokenized_datasets["test"])

4.9、模型预测

trainer.predict(tokenized_datasets["test"])

sen = '我觉得这家酒店不错,饭很好吃'
inputs = tokenizer(sen,return_tensors='pt')
id2label = {0:'差评!',1:'好评!'}
model.eval()
with torch.inference_mode():
    inputs = tokenizer(sen,return_tensors='pt')
    inputs = {k:v.cuda() for k,v in inputs.items()}
    logits = model(**inputs).logits
    pred = torch.argmax(logits,dim=-1)
    print(f'输入:{sen}\n模型预测结果:{pred.item()}')
    print(pred)

from transformers import pipeline

model.config.id2label = id2label
pipe = pipeline('text-classification',model=model,tokenizer=tokenizer,device=0)

pipe(sen)