Pytorch-DataLoader 和 Dataset

参考：
https://blog.csdn.net/zw__chen/article/details/82806900

https://blog.csdn.net/Threelights/article/details/88680540

一 要点总结

• 1 torch.utils.data.Dataset是一个包装类，用来将数据包装为Dataset类，然后传入DataLoader中。
• 2 将数据包装为Dataset类有两种方式：一种是用 torch.utils.data.Dataset.TensorDataset 来将数据包装成Dataset类，一种是自己写一个继承 torch.utils.data.Dataset类 ，实现类中的 len 方法和getitem 方法。
• 3 torchvision.datasets中的所有数据集都是 torch.utils.data.dataset 的子类，因而都实现了 getitem 和 len 方法，可以直接传递给 torch.utils.data.dataloader

二 示例

1 用 torch.utils.data.Dataset.TensorDataset 来将数据包装成Dataset类

import h5py
import numpy as np
from torch.utils.data import DataLoader, TensorDataset, Dataset

# 查看数据

train_dataset = h5py.File('datasets/train_signs.h5', "r")
train_dataset
# out: <HDF5 file "train_signs.h5" (mode r)>
train_dataset.keys()
# out: <KeysViewHDF5 ['list_classes', 'train_set_x', 'train_set_y']>
list_classes,train_set_x,train_set_y  = train_dataset['list_classes'],train_dataset['train_set_x'],train_dataset['train_set_y']
list_classes, train_set_x, train_set_y
# out: (<HDF5 dataset "list_classes": shape (6,), type "<i8">,
# out: <HDF5 dataset "train_set_x": shape (1080, 64, 64, 3), type "|u1">,
# out: <HDF5 dataset "train_set_y": shape (1080,), type "<i8">)

# 用 torch.utils.data.Dataset.TensorDataset 来将数据包装成Dataset类

x_train = np.array(train_dataset["train_set_x"][:]) # your train set features
x_train = np.transpose(x_train, (0, 3, 1, 2))
y_train = np.array(train_dataset["train_set_y"][:]) # your train set labels
y_train = y_train.reshape((1, y_train.shape[0])).T
X_train_tensor = torch.tensor(x_train, dtype=torch.float)/255
Y_train_tensor = torch.tensor(y_train, dtype=torch.long)
train_dataset = TensorDataset(X_train_tensor, Y_train_tensor)
train_loader = DataLoader(train_dataset, batch_size=64, shuffle=True)

for epoch in range(2):
    for i,data in enumerate(train_loader):
        inputs,labels = data
        print(epoch,i,"inputs",inputs.size(),"labels",labels.size())

2 继承 torch.utils.data.Dataset类 ，实现类中的 len 方法和getitem 方法

# 继承 Dataset类 
# 重写 len 方法，该方法提供了dataset的大小； getitem 方法， 该方法支持从 0 到 len(self)的索引
class TrainDataset(Dataset):
    """
        下载数据、初始化数据，都可以在这里完成
    """
    def __init__(self):
        train_dataset = h5py.File('datasets/train_signs.h5', "r") # 读取数据
        x_train,y_train = np.array(train_dataset["train_set_x"][:]),np.array(train_dataset["train_set_y"][:]) # 读取数据为array
        x_train,y_train = np.transpose(x_train, (0, 3, 1, 2)),y_train.reshape((1, y_train.shape[0])).T # 改变array格式
        X_train_tensor,Y_train_tensor = torch.tensor(x_train, dtype=torch.float)/255,torch.tensor(y_train, dtype=torch.long) # 将array转换为tensor
        self.x_data =  X_train_tensor
        self.y_data = Y_train_tensor
    
    def __getitem__(self, index):
        return self.x_data[index], self.y_data[index]

    def __len__(self):
        return x_train.shape[0] 
#  实例化这个类，就得到Dataset类型的数据，将这个类传给DataLoader
trainDataset =  TrainDataset()
type(trainDataset)
# out: __main__.TrainDataset
len(trainDataset)
# out: 1080
train_loader2 = DataLoader(dataset=trainDataset,
                          batch_size=64,
                          shuffle=True)

for epoch in range(2):
    for i, data in enumerate(train_loader2):
        inputs, labels = data
        # 接下来就是跑模型的环节了，我们这里使用print来代替
        print("epoch：", epoch, "的第" , i, "个inputs", inputs.size(), "labels", labels.size())

3 torchvision.datasets.mnist使用示例

from torchvision.datasets import mnist
from torch.utils.data import DataLoader
from torchvision.transforms import ToTensor

# 查看数据

train_dataset = mnist.MNIST(root='./train', train=True)
train_dataset
# out: Dataset MNIST
# out:    Number of datapoints: 60000
# out:    Root location: ./train
# out:    Split: Train
type(train_dataset)
# out: torchvision.datasets.mnist.MNIST
len(train_dataset)
# out: 60000
train_dataset[0]
# out: (<PIL.Image.Image image mode=L size=28x28 at 0x1323FD26D88>, 5)
train_dataset[0][1]
# out: 5

# 将这个类传给DataLoader

train_loader = DataLoader(dataset=train_dataset,batch_size=64,shuffle=True)
for epoch in range(2):
    for i, data in enumerate(train_loader):
        inputs, labels = data
        # 接下来就是跑模型的环节了，我们这里使用print来代替
        print("epoch：", epoch, "的第" , i, "个inputs", inputs.size(), "labels", labels.size())
# TypeError: default_collate: batch must contain tensors, numpy arrays, numbers, dicts or lists; found <class 'PIL.Image.Image'>
# 必须是tensors, numpy arrays, numbers, dicts or lists，这里是class 'PIL.Image.Image'

# DataLoader承接的Dataset类必须是tensors, numpy arrays, numbers, dicts or lists，因而用transform=ToTensor()将其转换为tensor
train_dataset = mnist.MNIST(root='./train', train=True,transform=ToTensor())
train_dataset
# out: Dataset MNIST
# out:    Number of datapoints: 60000
# out:    Root location: ./train
# out:    Split: Train
# out:    StandardTransform
# out:Transform: ToTensor()
type(train_dataset)
# out: torchvision.datasets.mnist.MNIST
len(train_dataset)
# out: 60000
train_dataset[0]
# out: 

train_loader = DataLoader(dataset=train_dataset,batch_size=64,shuffle=True)

for epoch in range(2):
    for i, data in enumerate(train_loader):
        inputs, labels = data
        # 接下来就是跑模型的环节了，我们这里使用print来代替
        print("epoch：", epoch, "的第" , i, "个inputs", inputs.size(), "labels", labels.size())