ValueError: too many dimensions ‘str‘

使用torchtext读取预处理好的文本csv文件时遇到了报错，这是我的代码

import torch
import torch.nn as nn
import torch.utils.data as Data
from torchtext import data

if __name__ == "__main__":
    mytokenize = lambda x : x.split()   # str -> list[str]

    # 创建预处理对象
    TEXT = data.Field(
        sequential=True, # 这是文本序列
        tokenize=mytokenize, # 分词函数
        include_lengths=True,
        use_vocab=True,  # 建立词表
        batch_first=True,   # 批优先
        fix_length=200  # 定长
    )

    LABEL  =data.Field(
        sequential=True,
        use_vocab=False,
        pad_token=None,
        unk_token=None
    )

    # 对列进行处理
    train_test_fields = [
        ("text", TEXT),
        ("label", LABEL)
    ]

    # 读取数据
    traindata, testdata = data.TabularDataset.splits(
        path="./data", format="csv",    # 数据所在目录
        train="imdb_train.csv", test="imdb_test.csv",
        fields=train_test_fields,
        skip_header=True
    )

    ex0 = traindata.examples[0]
    print(ex0.label)

    train_data, val_data  = traindata.split(split_ratio=0.7)

    # 将训练集转化为词向量
    TEXT.build_vocab(train_data, max_size=20000)
    LABEL.build_vocab(train_data)
    # 训练集中的前10个高频词
    print(TEXT.vocab.freqs.most_common(n=10))
    print(TEXT.vocab.itos[:10])

    BATCH_SIZE = 32
    # 定义加载器
    train_iter = data.BucketIterator(train_data, batch_size=BATCH_SIZE)
    val_iter = data.BucketIterator(val_data, batch_size=BATCH_SIZE)
    test_iter = data.BucketIterator(testdata, batch_size=BATCH_SIZE)

    for batch in train_iter:
        break

    print("data's label", batch.label)
    print("shape of data", batch.text[0].shape)
    print("number of data", len(batch.text[1]))

报错是ValueError: too many dimensions 'str'。这个错误可以回溯到tensor对象的生成。结论是split读入文件时，标签是以[“1”]这样的str列表读入的，但是生成词表时需要[1]这样的数字列表。
加上其他博客写的，遇到这个错误可能的原因是某些数据是数字类型的，却以str对象的形式传入torch.tensor中，记得转换。

解决方法

在定义预处理对象LABEL时加入额外的分词函数，这是修改后的：

import torch
import torch.nn as nn
import torch.utils.data as Data
from torchtext import data

if __name__ == "__main__":
    mytokenize = lambda x : x.split()   # str -> list[str]
    labeltokenize = lambda x : [int(x[0])]  # [str] -> [int]
    # 创建预处理对象
    TEXT = data.Field(
        sequential=True, # 这是文本序列
        tokenize=mytokenize, # 分词函数
        include_lengths=True,
        use_vocab=True,  # 建立词表
        batch_first=True,   # 批优先
        fix_length=200  # 定长
    )

    LABEL  =data.Field(
        sequential=True,
        use_vocab=False,
        dtype=torch.int64,
        tokenize=labeltokenize,
        pad_token=None,
        unk_token=None
    )

    # 对列进行处理
    train_test_fields = [
        ("text", TEXT),
        ("label", LABEL)
    ]

    # 读取数据
    traindata, testdata = data.TabularDataset.splits(
        path="./data", format="csv",    # 数据所在目录
        train="imdb_train.csv", test="imdb_test.csv",
        fields=train_test_fields,
        skip_header=True
    )

    ex0 = traindata.examples[0]
    print(ex0.label)

    train_data, val_data  = traindata.split(split_ratio=0.7)

    # 将训练集转化为词向量
    TEXT.build_vocab(train_data, max_size=20000)
    LABEL.build_vocab(train_data)
    # 训练集中的前10个高频词
    print(TEXT.vocab.freqs.most_common(n=10))
    print(TEXT.vocab.itos[:10])

    BATCH_SIZE = 32
    # 定义加载器
    train_iter = data.BucketIterator(train_data, batch_size=BATCH_SIZE)
    val_iter = data.BucketIterator(val_data, batch_size=BATCH_SIZE)
    test_iter = data.BucketIterator(testdata, batch_size=BATCH_SIZE)

    for batch in train_iter:
        break

    print("data's label", batch.label)
    print("shape of data", batch.text[0].shape)
    print("number of data", len(batch.text[1]))

输出：

[1]
[('movie', 29836), ('film', 26802), ('one', 17978), ('like', 13816), ('good', 10270), ('even', 8722), ('would', 8564), ('time', 8433), ('really', 8220), ('story', 8185)]
['<unk>', '<pad>', 'movie', 'film', 'one', 'like', 'good', 'even', 'would', 'time']
data's label tensor([[1, 1, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 0, 1, 1, 1, 0, 0,
         0, 0, 1, 0, 0, 0, 0, 1]])
shape of data torch.Size([32, 200])
number of data 32

done(｀・ω・´)