pytorch文本任务pipeline

---

https://pytorch.org/tutorials/beginner/text_sentiment_ngrams_tutorial.html
https://github.com/lankuohsing/pytorch-study/tree/main/utils/text_pipline

1. 读取文本数据制作成pytorch专用数据集

以读取CSV数据为例，里面每行的数据格式为：label,text。下面的代码会得到一个IterableDataset类型对象，可用于后面的流程

import io
from torchtext.utils import unicode_csv_reader
from torchtext.data.datasets_utils import _RawTextIterableDataset
def _create_data_from_csv(data_path):
    with io.open(data_path, encoding="utf8") as f:
        reader = unicode_csv_reader(f)
        for row in reader:
            yield int(row[0]), ' '.join(row[1:])
def _create_raw_text_iterable_dataset(description=None,path=None):
    iterator=_create_data_from_csv(path)
    length=sum(1 for _ in iterator)
    iterator=_create_data_from_csv(path)
#    print("lenght: ",length)
    return _RawTextIterableDataset(description=description,full_num_lines=length,iterator=iterator)
train_iter = _create_raw_text_iterable_dataset(description="AG_NEWS",path="../dataset/AG_NEWS/train.csv")#

2. 对句子进行分词得到词库

from gen_dataset_from_text import train_iter
from torchtext.vocab import build_vocab_from_iterator
from torchtext.data import get_tokenizer
# In[]
sentence="You can now install TorchText using pip!"
tokenizer = get_tokenizer(tokenizer=None, language='en')
#输入是字符串，输出是单词列表，不会自动分离标点
tokens = tokenizer(sentence)
print(tokens)

tokenizer = get_tokenizer(tokenizer='basic_english', language='en')

#输入是字符串，输出是单词列表，会自动分离标点
tokens = tokenizer(sentence)
print(tokens)


def yield_tokens(data_iter):
    for _, text in data_iter:
        yield tokenizer(text)
vocab = build_vocab_from_iterator(yield_tokens(train_iter), specials=["<unk>"])
vocab.set_default_index(vocab["<unk>"])
"""
vocab(["str"]):返回该单词的下标
vocabulary(["str1","str2"]):返回一个列表，包含每个单词的下标
"""

注意，参数tokenizer可以是其他分词函数。如果是None，则直接按空格切分（无法剔除紧挨着单词的标点符号）；如果是basic_english则会把标点符号单独拎出来，称为normalize

vocab的常用属性/方法

1. __len__() → int： 返回词库大小
2. __getitem__(token: str) → int: 给定token返回其下标，等价于vocab[str]
3. lookup_indices(self, tokens: List[str]) -> List[int]:返回单词列表里每个单词的下标，以一个数组的形式返回呈现，等价于vocab(List[str])
4. set_default_index(index: Optional[int]) → None：用于设置OOV对应的词和下标

3. 构建DataLoadar

from torchtext.vocab import build_vocab_from_iterator
from gen_dataset_from_text import train_iter
from tokenize_with_corpus import vocab,tokenizer
import torch

#构建两个比较有用的lambda表达式
text_pipeline = lambda x: vocab(tokenizer(x))#
label_pipeline = lambda x: int(x) - 1

# In[]
print(text_pipeline('here is the an example'))
print(label_pipeline('10'))
# In[]
from torch.utils.data import DataLoader
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

def collate_batch(batch):
    label_list, text_list, offsets = [], [], [0]
    for (_label, _text) in batch:
         label_list.append(label_pipeline(_label))
         processed_text = torch.tensor(text_pipeline(_text), dtype=torch.int64)
         text_list.append(processed_text)
         offsets.append(processed_text.size(0))
    label_list = torch.tensor(label_list, dtype=torch.int64)
    offsets = torch.tensor(offsets[:-1]).cumsum(dim=0)#为了后文的模型用
    text_list = torch.cat(text_list)
    return label_list.to(device), text_list.to(device), offsets.to(device)
# In[]
dataloader = DataLoader(train_iter, batch_size=8, shuffle=False, collate_fn=collate_batch)

4. 模型构建（待更新）