本篇文章参考：
Multi-class Sentiment Analysis
部分细节可能会略作改动，代码注释尽数基于自己的理解。文章目的仅作个人领悟记录，并不完全是tutorial的翻译，可能并不适用所有初学者，但也可从中互相借鉴吸收参考。
接上篇：torchtext使用–convolution IMDB
这是第五篇
这次我们将尝试多分类任务TREC而不是IMDB
import torch
import torchtext
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torchtext import data
from torchtext import datasets
import numpy as np
import random
import math
use_cuda=torch.cuda.is_available()
device=torch.device("cuda" if use_cuda else "cpu")
SEED=1234
random.seed(SEED)
np.random.seed(SEED)
torch.manual_seed(SEED)
if use_cuda:
torch.cuda.manual_seed(SEED)
torch.backends.cudnn.deterministic = True
torchtext使用的四步
由于这次是多label，因此对于LabelField，不用将其设置为float，因为下游算loss的函数不再是二元交叉熵而是crossentropy似然估计，label要求是LongTensor
1.构建Field
TEXT=data.Field(tokenize="spacy",tokenizer_language="en_core_web_sm",batch_first=True)
LABEL=data.LabelField()
2.构建datasets
由于TREC数据集可以选定参数fine_grained,让数据集是6分类还是50分类的，这里选择5分类。
train_data, test_data=datasets.TREC.splits(TEXT,LABEL,fine_grained=False)
train_data, valid_data=train_data.split(split_ratio=0.8)
查看一下：
vars(train_data[1])
{'text': ['Who',
'replaced',
'Bert',
'Parks',
'as',
'the',
'host',
'of',
'The',
'Miss',
'America',
'Pageant',
'?'],
'label': 'HUM'}
3.构建vocab
MAX_VOCAB_SIZE = 25_000
TEXT.build_vocab(train_data,max_size=MAX_VOCAB_SIZE,
vectors="glove.6B.100d",unk_init=torch.Tensor.normal_)
LABEL.build_vocab(train_data)
看一下标签：
LABEL.vocab.stoi
defaultdict(None,
{'HUM': 0, 'ENTY': 1, 'DESC': 2, 'NUM': 3, 'LOC': 4, 'ABBR': 5})
看一下数据集：
TEXT.vocab.itos[:10]
['<unk>', '<pad>', '?', 'the', 'What', 'is', 'of', 'in', 'a', '`']
4.创建iterator
BATCH_SIZE = 64
train_iterator, valid_iterator, test_iterator =data.BucketIterator.splits((train_data,valid_data,test_data),
batch_size=BATCH_SIZE,device=device)
创建模型
这里依旧使用之前用过的conv层，由于上篇使用的是二维，这里示范一维卷积。
其实一维卷积、二维卷积区别就在kernel_size的维度.在运用卷积的时候，你完全不用担心卷积窗口到底怎么卷，你只需要关心窗口大小和卷积步长，这决定了卷积结果的大小（seq - kernel_size + 1）.模型前向传播的时候会自动滑动窗口。
所以很多情况下，卷积的维度可以转换，只要将输入的维度转换得符合你的要求。
这里，将输入的embedding作为channel的维度，这样可以不用unsqueeze出一个channel维度。
另外，注意卷积之后不要忘了经过一层relu激活；喂给全连接层之前记得dropout失活。
from typing import Tuple
class CNN(nn.Module):
def __init__(self, vocab_size, embedding_dim, n_filters, filter_sizes:Tuple, output_dim,
dropout, pad_idx):
super(CNN, self).__init__()
self.embed=nn.Embedding(vocab_size,embedding_dim,pad_idx)
self.convs=nn.ModuleList([nn.Conv1d(in_channels=embedding_dim,out_channels=n_filters
,kernel_size=fs) for fs in filter_sizes])
self.fc=nn.Linear(len(filter_sizes)*n_filters,output_dim)
self.dropout=nn.Dropout(dropout)
def forward(self,text):
#text:(batch,seq)
embeded=self.embed(text)
#embeded:(batch,seq,embedding)
embeded=embeded.permute(0,2,1)
# embeded:(batch,embedding,seq)
conveds=[torch.relu(conv(embeded)) for conv in self.convs]
#conved:(batch,n_filters,seq-kernel_size+1)
pooled=[F.max_pool1d(conved,conved.shape[-1]).squeeze(-1) for conved in conveds]
#pooled:(batch,n_filters)
cat=self.dropout(torch.cat(pooled,dim=1))
#cat:(batch,n_filters*len(filter_size))
return self.fc(cat)
#（batch,output_dim）
实例化模型
INPUT_DIM = len(TEXT.vocab)
EMBEDDING_DIM = 100
N_FILTERS = 100
FILTER_SIZES = [2,3,4]
OUTPUT_DIM = len(LABEL.vocab)
DROPOUT = 0.5
PAD_IDX = TEXT.vocab.stoi[TEXT.pad_token]
model = CNN(INPUT_DIM, EMBEDDING_DIM, N_FILTERS, FILTER_SIZES, OUTPUT_DIM, DROPOUT, PAD_IDX)
数一下参数
def count_parameters(model):
return sum(p.numel() for p in model.parameters() if p.requires_grad)
print(f'The model has {count_parameters(model):,} trainable parameters')
The model has 903,606 trainable parameters
可以用下面的代码先将模型初始化
def init_parameters(model:nn.Module):
for name,p in model.named_parameters():
if 'weight' in name : #如果是权重，那就初始化为期望为0，标准差为0.01
nn.init.normal_(p.data,mean=0,std=0.01)
else: #如果是偏置常数，就设置为0
nn.init.constant_(p.data,0)
model.apply(init_parameters)
CNN(
(embed): Embedding(8115, 100, padding_idx=1)
(convs): ModuleList(
(0): Conv1d(100, 100, kernel_size=(2,), stride=(1,))
(1): Conv1d(100, 100, kernel_size=(3,), stride=(1,))
(2): Conv1d(100, 100, kernel_size=(4,), stride=(1,))
)
(fc): Linear(in_features=300, out_features=6, bias=True)
(dropout): Dropout(p=0.5, inplace=False)
)
设置embed初值
UNK_IDX = TEXT.vocab.stoi[TEXT.unk_token]
pretrained_embeddings = TEXT.vocab.vectors
model.embed.weight.data.copy_(pretrained_embeddings)
model.embed.weight.data[PAD_IDX]=torch.zeros(EMBEDDING_DIM)
model.embed.weight.data[UNK_IDX]=torch.zeros(EMBEDDING_DIM)
定义loss，optimizer，把训练的参数搬到GPU
这里loss_function用的是Crossentropy ，利用的是似然估计误差
criterion=nn.CrossEntropyLoss()
model=model.to(device)
criterion=criterion.to(device)
optimizer=optim.Adam(model.parameters())
定义acc计算
tensor.argmax(dim : int , keepdim : bool) 用于返回tensor中值最大的那个元素的下标。如果不指定dim，那么将是整个tensor的；keepdim用于指定返回的tensor是否保留，argmax之后余下的那个1的维度
tensor.eq(tensor) 用于比较两个形状像等的tensor各个位置是否相等，返回一个布尔tensor
from torch import Tensor
def categorical_accuracy(preds:Tensor, y:Tensor):
max_preds=preds.argmax(dim=1)
correct=max_preds.eq(y)
return correct.sum()/torch.FloatTensor([y.shape[0]])
定义训练
def train(model:nn.Module,iterator:data.BucketIterator,
optimizer:optim.Adam, criterion:nn.CrossEntropyLoss):
epoch_loss=0.
epoch_acc=0.
model.train()
for batch in iterator:
preds=model(batch.text)
loss=criterion(preds,batch.label)
acc=categorical_accuracy(preds,batch.label)
optimizer.zero_grad()
loss.backward()
optimizer.step()
epoch_loss+=loss.item()
epoch_acc+=acc.item()
return epoch_loss / len(iterator), epoch_acc / len(iterator)
def evaluate(model:nn.Module,iterator:data.BucketIterator,
criterion:nn.CrossEntropyLoss):
epoch_loss=0.
epoch_acc=0.
model.eval()
with torch.no_grad():
for batch in iterator:
preds=model(batch.text)
loss=criterion(preds,batch.label)
acc=categorical_accuracy(preds,batch.label)
epoch_loss+=loss.item()
epoch_acc+=acc.item()
return epoch_loss / len(iterator), epoch_acc / len(iterator)
开始训练
import time
def epoch_time(start_time, end_time):
elapsed_time = end_time - start_time
elapsed_mins = int(elapsed_time / 60)
elapsed_secs = int(elapsed_time - (elapsed_mins * 60))
return elapsed_mins, elapsed_secs
N_EPOCHS = 5
best_valid_loss = float('inf')
for epoch in range(N_EPOCHS):
start_time = time.time()
train_loss, train_acc = train(model, train_iterator, optimizer, criterion)
valid_loss, valid_acc = evaluate(model, valid_iterator, criterion)
end_time = time.time()
epoch_mins, epoch_secs = epoch_time(start_time, end_time)
if valid_loss < best_valid_loss:
best_valid_loss = valid_loss
torch.save(model.state_dict(), 'tut5-model.pt')
print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')
print(f'\tTrain Loss: {train_loss:.3f} | Train Acc: {train_acc*100:.2f}%')
print(f'\t Val. Loss: {valid_loss:.3f} | Val. Acc: {valid_acc*100:.2f}%')
Epoch: 01 | Epoch Time: 0m 2s
Train Loss: 1.266 | Train Acc: 50.20%
Val. Loss: 0.890 | Val. Acc: 65.89%
Epoch: 02 | Epoch Time: 0m 0s
Train Loss: 0.780 | Train Acc: 72.16%
Val. Loss: 0.622 | Val. Acc: 78.21%
Epoch: 03 | Epoch Time: 0m 0s
Train Loss: 0.545 | Train Acc: 81.20%
Val. Loss: 0.511 | Val. Acc: 81.68%
Epoch: 04 | Epoch Time: 0m 0s
Train Loss: 0.403 | Train Acc: 86.81%
Val. Loss: 0.452 | Val. Acc: 83.77%
Epoch: 05 | Epoch Time: 0m 0s
Train Loss: 0.295 | Train Acc: 90.39%
Val. Loss: 0.418 | Val. Acc: 84.64%
评估一下
model.load_state_dict(torch.load('tut5-model.pt'))
test_loss, test_acc = evaluate(model, test_iterator, criterion)
print(f'Test Loss: {test_loss:.3f} | Test Acc: {test_acc*100:.2f}%')
Test Loss: 0.325 | Test Acc: 88.40%
扫一扫
7573
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
