soft prompt 示例代码

主函数

import gc
import copy
import datetime
import time
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import torch
import torch.nn as nn
import os
import torch.optim as optim
from torch.optim import lr_scheduler
from torch.utils.data import Dataset, DataLoader
from torch.cuda import amp
import transformers
from transformers import BertTokenizer,BertForSequenceClassification, BertModel, BertConfig
from transformers import AdamW, get_linear_schedule_with_warmup
from tqdm import tqdm
from collections import defaultdict
import plotly.graph_objects as go
from sklearn.metrics import mean_squared_error
from data_floda import create_folds
from config import CONFIG
import warnings
warnings.filterwarnings("ignore")


class PROMPTEmbedding(nn.Module):
    def __init__(self,
                 wte: nn.Embedding,
                 n_tokens: int = 10,
                 random_range: float = 0.5,
                 initialize_from_vocab: bool = True):
        super(PROMPTEmbedding, self).__init__()
        self.wte = wte
        self.n_tokens = n_tokens
        self.learned_embedding = nn.parameter.Parameter(self.initialize_embedding(wte,
                                                                                  n_tokens,
                                                                                  random_range,
                                                                                  initialize_from_vocab))

    def initialize_embedding(self,
                             wte: nn.Embedding,
                             n_tokens: int = 10,
                             random_range: float = 0.5,
                             initialize_from_vocab: bool = True):
        if initialize_from_vocab:
            return self.wte.weight[:n_tokens].clone().detach()
        return torch.FloatTensor(wte.weight.size(1), n_tokens).uniform_(-random_range, random_range)

    def forward(self, tokens):
        input_embedding = self.wte(tokens[:, self.n_tokens:])
        learned_embedding = self.learned_embedding.repeat(input_embedding.size(0), 1, 1)
        return torch.cat([learned_embedding, input_embedding], 1)



df = pd.read_csv("../input/commonlitreadabilityprize/train.csv")
test_df = pd.read_csv("../input/commonlitreadabilityprize/test.csv",usecols=["id","excerpt"])
print('Number of training sentences: {:,}\n'.format(df.shape[0]))
df.sample(10)

def prep_text(text_df):
    text_df = text_df.str.replace("\n","",regex=False)
    return text_df.str.replace("\'s",r"s",regex=True).values
df["excerpt"] = prep_text(df["excerpt"])
test_df["excerpt"] = prep_text(test_df["excerpt"])



# create folds
df = create_folds(df, num_splits=5)



def set_seed(seed = CONFIG.seed):
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    torch.backends.cudnn.deterministic = True
    torch.backends.cudnn.benchmark = False
    os.environ['PYTHONHASHSEED'] = str(seed)
set_seed(CONFIG.seed)


class BERTDataset(Dataset):
    def __init__(self, df):
        self.text = df['excerpt'].values
        self.target = df['target'].values
        self.max_len = CONFIG.max_len
        self.tokenizer = CONFIG.tokenizer
        self.n_tokens = CONFIG.n_tokens

    def __len__(self):
        return len(self.text)

    def __getitem__(self, index):
        text = self.text[index]
        text = ' '.join(text.split())
        inputs = self.tokenizer.encode_plus(
            text,
            None,
            truncation=True,
            add_special_tokens=True,
            max_length=self.max_len,
            padding='max_length',
            return_token_type_ids=True
        )
        inputs['input_ids'] = torch.cat((torch.full((1, self.n_tokens), 500).resize(CONFIG.n_tokens),
                                         torch.tensor(inputs['input_ids'], dtype=torch.long)))
        inputs['attention_mask'] = torch.cat((torch.full((1, self.n_tokens), 1).resize(CONFIG.n_tokens),
                                              torch.tensor(inputs['attention_mask'], dtype=torch.long)))

        return {
            'ids': inputs['input_ids'],
            'mask': inputs['attention_mask'],

            'target': torch.tensor(self.target[index], dtype=torch.float)
        }

model = BertForSequenceClassification.from_pretrained(
    "bert-base-uncased",
    num_labels = 1,
    output_attentions = False,
    output_hidden_states = False,
)
prompt_emb = PROMPTEmbedding(model.get_input_embeddings(),
                      n_tokens=20,
                      initialize_from_vocab=True)
model.set_input_embeddings(prompt_emb)
model.cuda()


def get_data(fold):
    df_train = df[df.kfold != fold].reset_index(drop=True)
    df_valid = df[df.kfold == fold].reset_index(drop=True)

    train_dataset = BERTDataset(df_train)
    valid_dataset = BERTDataset(df_valid)

    train_loader = DataLoader(train_dataset, batch_size=CONFIG.train_batch,
                              num_workers=4, shuffle=True, pin_memory=True)
    valid_loader = DataLoader(valid_dataset, batch_size=CONFIG.valid_batch,
                              num_workers=4, shuffle=False, pin_memory=True)

    return train_loader, valid_loader


train_dataloader,validation_dataloader=get_data(0)




param_optimizer = list(model.named_parameters())
no_decay = ["bias", "LayerNorm.bias", "LayerNorm.weight"]
optimizer_parameters = [
    {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],
     'weight_decay': 0.0001},
    {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
     'weight_decay': 0.0}
    ]

optimizer = AdamW(optimizer_parameters, lr=CONFIG.learning_rate)


# Defining LR Scheduler
scheduler = get_linear_schedule_with_warmup(
    optimizer,
    num_warmup_steps=0,
    num_training_steps=len(train_dataloader)*CONFIG.epochs
)

lrs = []
for epoch in range(1, CONFIG.epochs + 1):
    if scheduler is not None:
        scheduler.step()
    lrs.append(optimizer.param_groups[0]["lr"])
layout = go.Layout(template= "plotly_dark",title='Learning_rate')
fig = go.Figure(layout=layout)

fig.add_trace(go.Scatter(x=list(range(CONFIG.epochs)), y=lrs,
                    mode='lines+markers',
                    name='Learning_rate'))
fig.show()


def loss_fn(output, target):
    return torch.sqrt(nn.MSELoss()(output, target))


def format_time(elapsed):
    elapsed_rounded = int(round((elapsed)))
    return str(datetime.timedelta(seconds=elapsed_rounded))


def run(model, optimizer, scheduler):
    set_seed(40)
    scaler = CONFIG.scaler
    training_stats = []
    total_t0 = time.time()
    best_rmse = np.inf
    epochs = CONFIG.epochs
    for epoch_i in range(0, epochs):
        print("")
        print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))
        print('Training...')
        t0 = time.time()
        total_train_loss = 0
        data_size = 0
        model.train()
        for step, batch in enumerate(train_dataloader):
            tr_loss = []
            b_input_ids = batch['ids'].to(CONFIG.device)
            b_input_mask = batch['mask'].to(CONFIG.device)
            b_labels = batch['target'].to(CONFIG.device)
            batch_size = b_input_ids.size(0)
            model.zero_grad()
            with amp.autocast(enabled=True):
                output = model(b_input_ids, attention_mask=b_input_mask)
                output = output["logits"].squeeze(-1)
                loss = loss_fn(output, b_labels)
                tr_loss.append(loss.item() / len(output))
            scheduler.step()
            scaler.scale(loss).backward()
            scaler.step(optimizer)
            scaler.update()
        avg_train_loss = np.mean(tr_loss)
        training_time = format_time(time.time() - t0)
        gc.collect()
        print("")
        print("  Average training loss: {0:.2f}".format(avg_train_loss))
        print("  Training epoch took: {:}".format(training_time))
        print("")
        print("Running Validation...")

        t0 = time.time()
        model.eval()
        val_loss = 0
        allpreds = []
        alltargets = []
        for batch in validation_dataloader:
            losses = []
            with torch.no_grad():
                device = CONFIG.device
                ids = batch["ids"].to(device)
                mask = batch["mask"].to(device)
                output = model(ids, mask)
                output = output["logits"].squeeze(-1)
                target = batch["target"].to(device)
                loss = loss_fn(output, target)
                losses.append(loss.item() / len(output))
                allpreds.append(output.detach().cpu().numpy())
                alltargets.append(target.detach().squeeze(-1).cpu().numpy())

        allpreds = np.concatenate(allpreds)
        alltargets = np.concatenate(alltargets)
        val_rmse = mean_squared_error(alltargets, allpreds, squared=False)
        losses = np.mean(losses)
        gc.collect()
        validation_time = format_time(time.time() - t0)
        print("  Validation Loss: {0:.2f}".format(losses))
        print("  Validation took: {:}".format(validation_time))

        if val_rmse <= best_rmse:
            print(f"Validation RMSE Improved ({best_rmse} -> {val_rmse})")
            best_rmse = val_rmse
            best_model_wts = copy.deepcopy(model.state_dict())
            PATH = "rmse{:.4f}_epoch{:.0f}.bin".format(best_rmse, epoch_i)
            torch.save(model.state_dict(), PATH)
            print("Model Saved")

        training_stats.append(
            {
                'epoch': epoch_i + 1,
                'Training Loss': avg_train_loss,
                'Valid. Loss': losses,
                'Training Time': training_time,
                'Validation Time': validation_time
            }
        )
    print("")
    print("Training complete!")
    return training_stats


def Visualizations(training_stats):
    pd.set_option('precision', 2)
    df_stats = pd.DataFrame(data=training_stats)
    df_stats = df_stats.set_index('epoch')
    layout = go.Layout(template= "plotly_dark")
    fig = go.Figure(layout=layout)
    fig.add_trace(go.Scatter(x=df_stats.index, y=df_stats['Training Loss'],
                    mode='lines+markers',
                    name='Training Loss'))
    fig.add_trace(go.Scatter(x=df_stats.index, y=df_stats['Valid. Loss'],
                    mode='lines+markers',
                    name='Validation Loss'))
    fig.show()



df=run(model,optimizer,scheduler)
Visualizations(df)

config 配置文件

class CONFIG:
    seed = 42
    max_len = 331
    train_batch = 16
    valid_batch = 32
    epochs = 10
    n_tokens=20
    learning_rate = 2e-5
    splits = 5
    scaler = amp.GradScaler()
    model='bert-base-cased'
    tokenizer = BertTokenizer.from_pretrained(model, do_lower_case=True)
    tokenizer.save_pretrained('./tokenizer')
    device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')

交叉验证

from sklearn.model_selection import StratifiedKFold, KFold
def create_folds(data, num_splits):
  # we create a new column called kfold and fill it with -1
  data["kfold"] = -1

  # the next step is to randomize the rows of the data
  data = data.sample(frac=1).reset_index(drop=True)

  # calculate number of bins by Sturge's rule
  # I take the floor of the value, you can also
  # just round it
  num_bins = int(np.floor(1 + np.log2(len(data))))

  # bin targets
  data.loc[:, "bins"] = pd.cut(
      data["target"], bins=num_bins, labels=False
  )

  # initiate the kfold class from model_selection module
  kf = StratifiedKFold(n_splits=num_splits)

  # fill the new kfold column
  # note that, instead of targets, we use bins!
  for f, (t_, v_) in enumerate(kf.split(X=data, y=data.bins.values)):
      data.loc[v_, 'kfold'] = f

  # drop the bins column
  data = data.drop("bins", axis=1)

  # return dataframe with folds
  return data