我的transformer

1.transformer实现文本分类

数据加载

# 导包
from transformers import AutoTokenizer, AutoModelForSequenceClassification
import pandas as pd

data = pd.read_csv("./data.csv")

构造DataLoader类

from torch.utils.data import Dataset

class MyDataset(Dataset):

    def __init__(self) -> None:
        super().__init__()
        self.data = pd.read_csv("./data.csv")
        self.data = self.data.dropna()

    def __getitem__(self, index):
        return self.data.iloc[index]["review"], self.data.iloc[index]["label"]
    
    def __len__(self):
        return len(self.data)

dataset = MyDataset()
for i in range(5):
    print(dataset[i])

"""
('距离川沙公路较近,但是公交指示不对,如果是"蔡陆线"的话,会非常麻烦.建议用别的路线.房间较为简单.', 1)
('商务大床房，房间很大，床有2M宽，整体感觉经济实惠不错!', 1)
('早餐太差，无论去多少人，那边也不加食品的。酒店应该重视一下这个问题了。房间本身很好。', 1)
('宾馆在小街道上，不大好找，但还好北京热心同胞很多~宾馆设施跟介绍的差不多，房间很小，确实挺小，但加上低价位因素，还是无超所值的；环境不错，就在小胡同内，安静整洁，暖气好足-_-||。。。呵还有一大优势就是从宾馆出发，步行不到十分钟就可以到梅兰芳故居等等，京味小胡同，北海距离好近呢。总之，不错。推荐给节约消费的自助游朋友~比较划算，附近特色小吃很多~', 1)
('CBD中心,周围没什么店铺,说5星有点勉强.不知道为什么卫生间没有电吹风', 1)
"""

数据集划分和构建

from torch.utils.data import random_split
from torch.utils.data import DataLoader
import torch

trainset, validset = random_split(dataset, lengths=[0.9, 0.1])
# 加载/rbt3文件夹下的分词器
tokenizer = AutoTokenizer.from_pretrained("rbt3")


def collate_func(batch):
    texts, labels = [], []
    for item in batch:
        texts.append(item[0])
        labels.append(item[1])
    inputs = tokenizer(texts, max_length=128, padding="max_length", truncation=True, return_tensors="pt")
    inputs["labels"] = torch.tensor(labels)
    return inputs

trainloader = DataLoader(trainset, batch_size=32, shuffle=True, collate_fn=collate_func)
validloader = DataLoader(validset, batch_size=64, shuffle=False, collate_fn=collate_func)

开始训练

from torch.optim import Adam

model = AutoModelForSequenceClassification.from_pretrained("./rbt3/")
optimizer = Adam(model.parameters(), lr=2e-5)

def evaluate():
    model.eval()
    acc_num = 0
    with torch.inference_mode():
        for batch in validloader:
            if torch.cuda.is_available():
                batch = {k: v.cuda() for k, v in batch.items()}
            output = model(**batch)
            pred = torch.argmax(output.logits, dim=-1)
            acc_num += (pred.long() == batch["labels"].long()).float().sum()
    return acc_num / len(validset)

global_step = 0
# 训练30次
for ep in range(30):
    model.train()
    for batch in trainloader:
        if torch.cuda.is_available():
            batch = {k: v.cuda() for k, v in batch.items()}
        optimizer.zero_grad()
        output = model(**batch)
        output.loss.backward()
        optimizer.step()
        if global_step % 100 == 0:
            print(f"ep: {ep}, global_step: {global_step}, loss: {output.loss.item()}")
        global_step += 1
    acc = evaluate()
    print(f"ep: {ep}, acc: {acc}")