关于新闻主题分类任务:目前视频和网上的代码都不能完整的运行,所以从下载数据集开始,重新写一下。
1. 数据集介绍
AG_NEWS 数据集包含4个文件,如下图
classes.txt:保存类别
test.csv:测试数据,7600条
train.csv:训练数据,120000条
2. 对数据集处理
导入包
import torch
import pandas as pd
from keras.preprocessing.text import Tokenizer
from keras.utils import pad_sequences
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data import DataLoader
import time
from torch.utils.data.dataset import random_split # 导入数据随机划分方法工具
import warnings
warnings.filterwarnings('ignore')
读取csv文件
def load_data(csv_file):
df = pd.read_csv(csv_file, header=None) # pd默认第一行不读取,所以添加 header
dataTmep = []
# 逐行读取,_ 行号,row 内容
for _, row in df.iterrows():
label = row[0]
context = row[1] + row[2] # 将标题,内容合并
dataTmep.append((label, context))
return dataTmep
cutlen = 64
train_dataset = load_data("./data/ag_news_csv/train.csv")
test_dataset = load_data("./data/ag_news_csv/test.csv")
将读取到的文件打包,形成可以读取的dataset,并生成vocab,查看结果
def process_datasets_by_Tokenizer(train_datasets, test_datasets, cutlen=cutlen):
tokenizer = Tokenizer()
train_datasets_texts = []
train_datasets_labels = []
test_datasets_texts = []
test_datasets_labels = []
for index in range(len(train_datasets)):
train_datasets_labels.append(train_datasets[index][0] - 1)
train_datasets_texts.append(train_datasets[index][1])
for index in range(len(test_datasets)):
test_datasets_labels.append(test_datasets[index][0] - 1)
test_datasets_texts.append(test_datasets[index][1])
all_datasets_texts = train_datasets_texts + test_datasets_texts
all_datasets_labels = train_datasets_labels + test_datasets_labels
tokenizer.fit_on_texts(all_datasets_texts)
train_datasets_seqs = tokenizer.texts_to_sequences(train_datasets_texts)
test_datasets_seqs = tokenizer.texts_to_sequences(test_datasets_texts)
train_datasets_seqs = pad_sequences(train_datasets_seqs, cutlen)
test_datasets_seqs = pad_sequences(test_datasets_seqs, cutlen)
train_datasets = list(zip(train_datasets_seqs, train_datasets_labels))
test_datasets = list(zip(test_datasets_seqs, test_datasets_labels))
vocab_size = len(tokenizer.index_word.keys())
num_class = len(set(all_datasets_labels))
return train_datasets, test_datasets, vocab_size, num_class, tokenizer
train_datasets, test_datasets, vocab_size, num_class, tokenizer = process_datasets_by_Tokenizer(train_dataset, test_dataset, cutlen=cutlen)
print("查看处理之后的数据: ")
print("train:\n", train_datasets[:2])
print("test:\n", test_datasets[:2])
print("vocab_size = {}, num_class = {}".format(vocab_size, num_class))
print()
3. 构建带有 Embedding 层的文本分类模型
BATCH_SIZE = 16
VOCAB_SIZE = vocab_size # 获得整个语料包含的不同词汇总数
NUM_CLASS = num_class # 获得类别总数
EMBED_DIM = 128
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
class TextSentiment(nn.Module):
def __init__(self, vocab_size, embed_dim, num_class):
"""
类的初始化函数
:param vocab_size: 整个语料包含的不同词汇总数
:param embed_dim: 指定词嵌入的维度
:param num_class: 文本分类的类别总数
"""
super().__init__()
self.embedding = nn.Embedding(vocab_size, embed_dim, sparse=True)
self.fc = nn.Linear(embed_dim, num_class)
self.init_weights()
def init_weights(self):
initrange = 0.5
self.embedding.weight.data.uniform_(-initrange, initrange)
self.fc.weight.data.uniform_(-initrange, initrange)
self.fc.bias.data.zero_()
def forward(self, text):
"""
逻辑函数
:param text: 文本数值映射后的结果
:return: 与类别数尺寸相同的张量,用以判断文本类别
"""
embedded = self.embedding(text)
c = embedded.size(0) // BATCH_SIZE
embedded = embedded[: BATCH_SIZE * c]
embedded = embedded.transpose(1, 0).unsqueeze(0)
embedded = F.avg_pool1d(embedded, kernel_size=c)
return self.fc(embedded[0].transpose(1, 0))
# 实例化模型
model = TextSentiment(VOCAB_SIZE + 1, EMBED_DIM, NUM_CLASS).to(device)
print("查看模型: ")
print(model)
print()
4. 对数据进行 batch 处理
def generate_batch(batch):
"""
生成 batch 数据函数
:param batch: 由样本核对应标签的元组组成的 batch_size 大小的列表,形如[(sample1, label1), (sample2, label2)......]
:return: 样本张量核标签各自的列表形式 (张量),形如 text = tensor([sample1, sample2....]),label = tensor([label1, label2,...])
"""
text = []
label = []
for item in batch:
text.extend(item[0])
label.append(item[1])
return torch.tensor(text), torch.tensor(label)
# 假设一个输入
print("测试将一个 batch 张量合并: ")
batch = [(torch.tensor([3, 23, 2, 8]), 1), (torch.tensor([3, 45, 21, 6]), 0)]
res = generate_batch(batch)
print(res)
print()
5.构建训练与验证函数
构建损失函数,优化器等
criterion = torch.nn.CrossEntropyLoss().to(device) # 选择损失函数,选择预定义的交叉熵损失函数
optimizer = torch.optim.SGD(model.parameters(), lr=4.0) # 选择随机梯度下降优化器
scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1, gamma=0.9) # 选择优化器步长调节方法 StepLR,用来衰减学习率
定义训练函数
def train(train_data):
train_loss = 0
train_acc = 0
# 使用数据加载器生成 BATCH_SIZE 大小的数据进行批次训练
data = DataLoader(train_data, batch_size=BATCH_SIZE, shuffle=True, collate_fn=generate_batch)
for i, (text, cls) in enumerate(data):
optimizer.zero_grad()
text = text.to(device)
cls = cls.to(device)
output = model(text)
loss = criterion(output, cls)
train_loss += loss.item() # 将该批次的损失加到总损失中
loss.backward()
optimizer.step()
train_acc += (output.argmax(1) == cls).sum().item() # 将该批次的准去率加到总准确率中 (返回 1 和 0,再被累加)
scheduler.step()
# 返回本轮训练的平均损失核平均准确率
return train_loss / len(train_data), train_acc / len(train_data)
定义预测函数
def valid(test_data):
loss = 0
acc = 0
# 和训练相同,使用 DataLoader 获得训练数据生成器
data = DataLoader(test_data, batch_size=BATCH_SIZE, collate_fn=generate_batch)
for text, cls in data:
with torch.no_grad():
text = text.to(device)
cls = cls.to(device)
output = model(text)
loss = criterion(output, cls)
loss += loss.item() # 将损失和准确率加到总损失和准确率中
acc += (output.argmax(1) == cls).sum().item()
# 返回本轮验证的平均损失和平均准确率
return loss / len(test_data), acc / len(test_data)
6. 进行模型训练和验证(调用已经定义的模块)
定义训练信息
N_EPOCHS = 20 # 指定训练轮数
train_len = int(len(train_datasets) * 0.95) # 从 train_datasets 取出 0.95 作为训练集,先取其长度
# 然后使用 random_split 进行乱序划分,得到对应的训练集和验证集
sub_train_, sub_valid_ = random_split(train_datasets, [train_len, len(train_datasets) - train_len])
迭代训练,并打印训练集、验证集的损失函数和准确率
# 开始每一轮训练
for epoch in range(N_EPOCHS):
start_time = time.time() # 记录训练开始的时间
# 调用 train 和 valid 函数得到训练和验证的平均损失,平均准确率
train_loss, train_acc = train(sub_train_)
valid_loss, valid_acc = valid(sub_valid_)
# 计算训练和验证的总耗时
secs = int(time.time() - start_time)
# 用分钟和秒表示
mins = secs / 60
secs = secs % 60
# 打印训练和验证耗时,平均损失,平均准确率
print('Epoch: %d' % (epoch + 1), " | time in %d minutes, %d seconds" % (mins, secs))
print(f'\t Loss: {train_loss: .4f}(train) \t | \t Acc: {train_acc * 100: .1f} % (train)')
print(f'\t Loss: {valid_loss: .4f}(valid) \t | \t Acc: {valid_acc * 100: .1f} % (valid)')
7. 用测试集进行测试
valid_loss, valid_acc = valid(test_datasets)
print("测试集上测试: ")
print(f'\t Loss: {valid_loss: .4f}(valid) \t | \t Acc: {valid_acc * 100: .1f} % (valid)')
print()