Torchtext处理Fasttext实例

链接：https://pan.baidu.com/s/17IhfiYTfo0UeLoir1h3kjA

提取码：px2z

import torch
import torch.nn as nn
import torch.optim as optim

from tqdm import tqdm
from torchtext import data
from torchtext.vocab import Vectors
from torchtext.data import Iterator,BucketIterator,TabularDataset

import os
import random
import logging
import numpy as np
import pandas as pd


device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# 返回需要的Dataset用于构造迭代器
# 数据格式如下，每一条数据有三列，第一列为标签，第二列为title，第三列为content：
class MyDataset(data.Dataset):
    def __init__(self, path, text_field, label_field, test=False, aug=False, **kwargs):
        fields = [("label",label_field), ("title",None), ("text",text_field)]
        examples = []
        csv_data = pd.read_csv(path)
        print('read data from {}'.format(path))
        if test:
            # 如果为测试集，加载label用于验证测试集准确率，否则在传入参数时也需将label设为None
            for label,text in tqdm(zip(csv_data['label'], csv_data['text'])):
                examples.append(data.Example.fromlist([label,None,text],fields))
                #print(vars(examples[len(examples)-1]))
        else:
            for label,text in tqdm(zip(csv_data['label'], csv_data['text'])):
                if aug:
                    rate = random.random()
                    if rate > 0.5: