BIlstm训练中文

最新推荐文章于 2023-07-11 13:09:26 发布

dada33dada

最新推荐文章于 2023-07-11 13:09:26 发布

阅读量422

点赞数

文章标签： pytorch

本文链接：https://blog.csdn.net/dada33dada/article/details/118314455

版权

本文档详细介绍了如何运用PyTorch框架，构建双向LSTM（Bi-LSTM）模型来处理和理解中文文本数据。通过实例演示了数据预处理、模型构建、训练过程以及评估指标，帮助读者掌握Bi-LSTM在中文自然语言处理中的应用。

摘要由CSDN通过智能技术生成

from torch.utils.data import DataLoader
import os
from torch.utils import data
import pandas as pd
import torch
import numpy as np
import re
from tqdm import tqdm
import pickle
from torch import nn
import torch.nn.functional as F
from torch import optim

max_len = 250
train_batch_size = 64
test_batch_size = 500
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

class ESISADataset(data.Dataset):
    def __init__(self, csv_file):
        self.data = pd.read_csv(csv_file)

    def __len__(self):
        return self.data.shape[0]

    def __getitem__(self, idx):
        item = self.data.iloc[idx]
        text = item["text"]
        label = item["label"]

        return text, label

# 读取数据
train_dataset = ESISADataset("data/SMP2019_ECISA/train.csv")
test_dataset = ESISADataset("data/SMP2019_ECISA/test.csv")
dev_dataset = ESISADataset("data/SMP2019_ECISA/dev.csv")

def tokenize(text):

    fileters = ['!', '"', '#', '$', '%', '&', '\(', '\)', '\*', '\+', ',', '-', '\.', '/', ':', ';', '<', '=', '>',
                '\?', '@', '\[', '\\', '\]', '^', '_', '`', '\{', '\|', '\}', '~', '\t', '\n', '\x97', '\x96', '”',
                '“', ]
    text = re.sub("<.*?>", " ", text, flags=re.S)
    text = re.sub("|".join(fileters), " ", text, flags=re.S)
    return [i.strip() for i in text.split()]

"""文本序列化"""
# Word2Sequence
class Word2S