主函数
import gc
import copy
import datetime
import time
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import torch
import torch.nn as nn
import os
import torch.optim as optim
from torch.optim import lr_scheduler
from torch.utils.data import Dataset, DataLoader
from torch.cuda import amp
import transformers
from transformers import BertTokenizer,BertForSequenceClassification, BertModel, BertConfig
from transformers import AdamW, get_linear_schedule_with_warmup
from tqdm import tqdm
from collections import defaultdict
import plotly.graph_objects as go
from sklearn.metrics import mean_squared_error
from data_floda import create_folds
from config import CONFIG
import warnings
warnings.filterwarnings("ignore")
class PROMPTEmbedding(nn.Module):
def __init__(self,
wte: nn.Embedding,
n_tokens: int = 10,
random_range: float = 0.5,
initialize_from_vocab: bool = True):
super(PROMPTEmbedding, self).__init__()
self.wte = wte
self.n_tokens = n_tokens
self.learned_embedding = nn.parameter.Parameter(self.initialize_embedding(wte,
n_tokens,
random_range,
initialize_from_vocab))
def initialize_embedding(self,
wte: nn.Embedding,
n_tokens: int = 10,
random_range: float = 0.5,
initialize_from_vocab: bool = True):
if initialize_from_vocab:
return self.wte.weight[:n_tokens].clone().detach()
return torch.FloatTensor(wte.weight.size(1), n_tokens).uniform_(-random_range, random_range)
def forward(self, tokens):
input_embedding = self.wte(tokens[:, self.n_tokens:])
learned_embedding = self.learned_embedding.repeat(input_embedding.size(0), 1, 1)
return torch.cat([learned_embedding, input_embedding], 1)
df = pd.read_csv("../input/commonlitreadabilityprize/train.csv")
test_df = pd.read_csv("../input/commonlitreadabilityprize/test.csv",usecols=["id","excerpt"])
print('Number of training sentences: {:,}\n'.format(df.shape[0]))
df.sample(10)
def prep_text(text_df):
text_df = text_df.str.replace("\n","",regex=False)
return text_df.str.replace("\'s",r"s",regex=True).values
df["excerpt"] = prep_text(df["excerpt"])
test_df["excerpt"] = prep_text(test_df["excerpt"])
# create folds
df = create_folds(df, num_splits=5)
def set_seed(seed = CONFIG.seed):
np.random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed(seed)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
os.environ['PYTHONHASHSEED'] = str(seed)
set_seed(CONFIG.seed)
class BERTDataset(Dataset):
def __init__(self, df):
self.text = df['excerpt'].values
self.target = df['target'].values
self.max_len = CONFIG.max_len
self.tokenizer = CONFIG.tokenizer
self.n_tokens = CONFIG.n_tokens
def __len__(self):
return len(self.text)
def __getitem__(self, index):
text = self.text[index]
text = ' '.join(text.split())
inputs = self.tokenizer.encode_plus(
text,
None,
truncation=True,
add_special_tokens=True,
max_length=self.max_len,
padding='max_length',
return_token_type_ids=True
)
inputs['input_ids'] = torch.cat((torch.full((1, self.n_tokens), 500).resize(CONFIG.n_tokens),
torch.tensor(inputs['input_ids'], dtype=torch.long)))
inputs['attention_mask'] = torch.cat((torch.full((1, self.n_tokens), 1).resize(CONFIG.n_tokens),
torch.tensor(inputs['attention_mask'], dtype=torch.long)))
return {
'ids': inputs['input_ids'],
'mask': inputs['attention_mask'],
'target': torch.tensor(self.target[index], dtype=torch.float)
}
model = BertForSequenceClassification.from_pretrained(
"bert-base-uncased",
num_labels = 1,
output_attentions = False,
output_hidden_states = False,
)
prompt_emb = PROMPTEmbedding(model.get_input_embeddings(),
n_tokens=20,
initialize_from_vocab=True)
model.set_input_embeddings(prompt_emb)
model.cuda()
def get_data(fold):
df_train = df[df.kfold != fold].reset_index(drop=True)
df_valid = df[df.kfold == fold].reset_index(drop=True)
train_dataset = BERTDataset(df_train)
valid_dataset = BERTDataset(df_valid)
train_loader = DataLoader(train_dataset, batch_size=CONFIG.train_batch,
num_workers=4, shuffle=True, pin_memory=True)
valid_loader = DataLoader(valid_dataset, batch_size=CONFIG.valid_batch,
num_workers=4, shuffle=False, pin_memory=True)
return train_loader, valid_loader
train_dataloader,validation_dataloader=get_data(0)
param_optimizer = list(model.named_parameters())
no_decay = ["bias", "LayerNorm.bias", "LayerNorm.weight"]
optimizer_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],
'weight_decay': 0.0001},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
'weight_decay': 0.0}
]
optimizer = AdamW(optimizer_parameters, lr=CONFIG.learning_rate)
# Defining LR Scheduler
scheduler = get_linear_schedule_with_warmup(
optimizer,
num_warmup_steps=0,
num_training_steps=len(train_dataloader)*CONFIG.epochs
)
lrs = []
for epoch in range(1, CONFIG.epochs + 1):
if scheduler is not None:
scheduler.step()
lrs.append(optimizer.param_groups[0]["lr"])
layout = go.Layout(template= "plotly_dark",title='Learning_rate')
fig = go.Figure(layout=layout)
fig.add_trace(go.Scatter(x=list(range(CONFIG.epochs)), y=lrs,
mode='lines+markers',
name='Learning_rate'))
fig.show()
def loss_fn(output, target):
return torch.sqrt(nn.MSELoss()(output, target))
def format_time(elapsed):
elapsed_rounded = int(round((elapsed)))
return str(datetime.timedelta(seconds=elapsed_rounded))
def run(model, optimizer, scheduler):
set_seed(40)
scaler = CONFIG.scaler
training_stats = []
total_t0 = time.time()
best_rmse = np.inf
epochs = CONFIG.epochs
for epoch_i in range(0, epochs):
print("")
print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))
print('Training...')
t0 = time.time()
total_train_loss = 0
data_size = 0
model.train()
for step, batch in enumerate(train_dataloader):
tr_loss = []
b_input_ids = batch['ids'].to(CONFIG.device)
b_input_mask = batch['mask'].to(CONFIG.device)
b_labels = batch['target'].to(CONFIG.device)
batch_size = b_input_ids.size(0)
model.zero_grad()
with amp.autocast(enabled=True):
output = model(b_input_ids, attention_mask=b_input_mask)
output = output["logits"].squeeze(-1)
loss = loss_fn(output, b_labels)
tr_loss.append(loss.item() / len(output))
scheduler.step()
scaler.scale(loss).backward()
scaler.step(optimizer)
scaler.update()
avg_train_loss = np.mean(tr_loss)
training_time = format_time(time.time() - t0)
gc.collect()
print("")
print(" Average training loss: {0:.2f}".format(avg_train_loss))
print(" Training epoch took: {:}".format(training_time))
print("")
print("Running Validation...")
t0 = time.time()
model.eval()
val_loss = 0
allpreds = []
alltargets = []
for batch in validation_dataloader:
losses = []
with torch.no_grad():
device = CONFIG.device
ids = batch["ids"].to(device)
mask = batch["mask"].to(device)
output = model(ids, mask)
output = output["logits"].squeeze(-1)
target = batch["target"].to(device)
loss = loss_fn(output, target)
losses.append(loss.item() / len(output))
allpreds.append(output.detach().cpu().numpy())
alltargets.append(target.detach().squeeze(-1).cpu().numpy())
allpreds = np.concatenate(allpreds)
alltargets = np.concatenate(alltargets)
val_rmse = mean_squared_error(alltargets, allpreds, squared=False)
losses = np.mean(losses)
gc.collect()
validation_time = format_time(time.time() - t0)
print(" Validation Loss: {0:.2f}".format(losses))
print(" Validation took: {:}".format(validation_time))
if val_rmse <= best_rmse:
print(f"Validation RMSE Improved ({best_rmse} -> {val_rmse})")
best_rmse = val_rmse
best_model_wts = copy.deepcopy(model.state_dict())
PATH = "rmse{:.4f}_epoch{:.0f}.bin".format(best_rmse, epoch_i)
torch.save(model.state_dict(), PATH)
print("Model Saved")
training_stats.append(
{
'epoch': epoch_i + 1,
'Training Loss': avg_train_loss,
'Valid. Loss': losses,
'Training Time': training_time,
'Validation Time': validation_time
}
)
print("")
print("Training complete!")
return training_stats
def Visualizations(training_stats):
pd.set_option('precision', 2)
df_stats = pd.DataFrame(data=training_stats)
df_stats = df_stats.set_index('epoch')
layout = go.Layout(template= "plotly_dark")
fig = go.Figure(layout=layout)
fig.add_trace(go.Scatter(x=df_stats.index, y=df_stats['Training Loss'],
mode='lines+markers',
name='Training Loss'))
fig.add_trace(go.Scatter(x=df_stats.index, y=df_stats['Valid. Loss'],
mode='lines+markers',
name='Validation Loss'))
fig.show()
df=run(model,optimizer,scheduler)
Visualizations(df)
config 配置文件
class CONFIG:
seed = 42
max_len = 331
train_batch = 16
valid_batch = 32
epochs = 10
n_tokens=20
learning_rate = 2e-5
splits = 5
scaler = amp.GradScaler()
model='bert-base-cased'
tokenizer = BertTokenizer.from_pretrained(model, do_lower_case=True)
tokenizer.save_pretrained('./tokenizer')
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
交叉验证
from sklearn.model_selection import StratifiedKFold, KFold
def create_folds(data, num_splits):
# we create a new column called kfold and fill it with -1
data["kfold"] = -1
# the next step is to randomize the rows of the data
data = data.sample(frac=1).reset_index(drop=True)
# calculate number of bins by Sturge's rule
# I take the floor of the value, you can also
# just round it
num_bins = int(np.floor(1 + np.log2(len(data))))
# bin targets
data.loc[:, "bins"] = pd.cut(
data["target"], bins=num_bins, labels=False
)
# initiate the kfold class from model_selection module
kf = StratifiedKFold(n_splits=num_splits)
# fill the new kfold column
# note that, instead of targets, we use bins!
for f, (t_, v_) in enumerate(kf.split(X=data, y=data.bins.values)):
data.loc[v_, 'kfold'] = f
# drop the bins column
data = data.drop("bins", axis=1)
# return dataframe with folds
return data