#pytorch创建数据集方法
#1.加载torchvision.datasets中自带数据集如MNIST,CIFAR等数据集,并创建数据迭代器
#2.使用torch.utils.data.TensorDataset根据Tensor创建数据集(numpy的array,pandas的DataFrame需要先转换成Tensor)
#3.使用torchvision.datasets.ImageFolder根据图片目录创建图片数据集
#4.使用torch.utils.data.Dataset创建自定义数据集
#此外还可通过torch.utils.data.random_split将一个数据集分割成多份,常用于分割训练集、验证集和测试集
#1.加载torchvision.datasets中自带数据集如MNIST,CIFAR等数据集,并创建数据迭代器
import torch
import torchvision
import matplotlib.pyplot as plt
import numpy as np
import torchvision.transforms as transforms
batch_size = 10
num_workers = 4
mnist_train = torchvision.datasets.FashionMNIST(root = "D:\\02Learning\\01_BioMedicalAI\\05_datasets\\001_mnist\\",
train=True,
download=True,
transform=transforms.ToTensor())
mnist_test = torchvision.datasets.FashionMNIST(root = "D:\\02Learning\\01_BioMedicalAI\\05_datasets\\001_mnist\\",
train=False,
download=True,
transform=transforms.ToTensor())
train_iter = torch.utils.data.DataLoader(mnist_train, batch_size=batch_size, shuffle=True,
num_workers=num_workers)
test_iter = torch.utils.data.DataLoader(mnist_test, batch_size=batch_size, shuffle=True,
num_workers=num_workers)
#2.使用torch.utils.data.TensorDataset根据Tensor创建数据集(numpy的array,pandas的DataFrame需要先转换成Tensor)
#根据Tensor创建数据集
#torch.utils.data.TensorDataset用来对tensor进行打包,功能类似Python的zip函数
import numpy as np
import torch
from torch.utils.data import TensorDataset, DataLoader, random_split
#根据tensor创建数据集
from sklearn import datasets
iris = datasets.load_iris()
ds_iris = TensorDataset(torch.tensor(iris.data), torch.tensor(iris.target))
#分割成训练集和测试集
n_train = int(len(ds_iris) * 0.8)
n_valid = len(ds_iris) - n_train
ds_train, ds_valid = random_split(ds_iris, [n_train, n_valid])
print(type(ds_iris))
print(type(ds_train))
#使用DataLoader加载数据集
dl_train, dl_valid = DataLoader(ds_train, batch_size=8), DataLoader(ds_valid, batch_size=8)
for features, labels in dl_train:
print(features, labels)
break
#3.使用torchvision.datasets.ImageFolder根据图片目录创建图片数据集
#(2)使用torchvision.datasets.ImageFolder根据图片目录创建图片数据集
#ImageFolder是一个通用的数据加载器
#ImageFolder(root, transform=None, target_transform=None, loader=<function default_loader>,
#is_valid_file=None)
#root指定路径加载图片,图片存储的根目录,即各文件夹所在目录的上一级目录
#transform对图片进行预处理操作,原始图片为输入,返回一个转换后的图片
#transform对PIL Image进行转换操作,
#target_transform对label的转换,输入为target,输出对其的转换
#loader给定路径后如何读取图片,默认读取RGB格式的PIL Image对象
import numpy as np
import torch
from torch.utils.data import DataLoader
import torchvision.datasets
#输入训练和测试集的路径
train_dir = "D:\\02Learning\\01_BioMedicalAI\\05_DatasetLib\\002_test\\01_train_dir\\"
test_dir = "D:\\02Learning\\01_BioMedicalAI\\05_DatasetLib\\002_test\\02_test_dir\\"
#将文件夹的内容载入dataset
train_dataset = torchvision.datasets.ImageFolder(train_dir)
test_dataset = torchvision.datasets.ImageFolder(test_dir)
#DataLoader数据
train_loader = DataLoader(
dataset = train_dataset, #输入要加载的数据集
batch_size = 4, #一个批量的大小
shuffle=True, #是否打乱顺序
num_workers = 4
)
test_loader = DataLoader(
dataset = test_dataset, #输入要加载的数据集
batch_size = 4, #一个批量的大小
shuffle=True,
num_workers = 4
)
#返回dataset有三种属性
#self.classes:
print(train_dataset.classes)
print(train_dataset.class_to_idx)
# print(train_dataset.imgs)
#4.使用torch.utils.data.Dataset创建自定义数据集
#torch.utils.data.Dataset与torch.utils.data.DataLoader组合得到数据迭代器,在每次训练时,
#利用该迭代器输出每一个batch数据,并能在输出时对数据进行相应的预处理或数据增强
#torch.utils.data.Dataset代表自定义数据集方法的类,用户可通过继承该类来自定义自己的数据集
#类,在继承时要求用户重载__len__()和__getitem__()这两个魔法方法
#torch.utils.data.DataLoader()为数据加载器,结合数据集和取样器,并提供多线程处理数据集
import os
from torchvision.io import read_image
import pandas as pd
from torch.utils.data import Dataset, DataLoader
import matplotlib.pyplot as plt
import numpy as np
import torch
from torchvision import transforms
class MyCarDataset(Dataset):
def __init__(self, img_dir, img_label_dir, transform=None):
super().__init__()
self.img_dir = img_dir
self.img_labels = pd.read_excel(img_label_dir)
self.transform = transform
def __len__(self):
return len(self.img_labels)
def __getitem__(self, index):
img_path = os.path.join(self.img_dir + self.img_labels.iloc[index, 0])
image = read_image(img_path)
label = self.img_labels.iloc[index, 1]
if self.transform is not None:
image = self.transform(image)
return image, label
def tensorToimg(img_tensor):
img = img_tensor.numpy()
img = np.transpose(img, [1, 2, 0])
plt.imshow(img)
plt.show()
label_dic = {0: 'byd', 1: 'McLaren'}
label_path = "D:\\02Learning\\01_BioMedicalAI\\02_paper_with_code\\pytorch_frame\\01_data_load_try_dataset\\label\\label.xlsx"
img_root_path = "D:\\02Learning\\01_BioMedicalAI\\02_paper_with_code\\pytorch_frame\\01_data_load_try_dataset\\img\\"
dataset = MyCarDataset(img_root_path, label_path)
image, label = dataset.__getitem__(1)
#图片变为tensor,维度为[通道数,长,宽]
tensorToimg(image)
dataloader = DataLoader(dataset, batch_size=1, shuffle=True)
#dataloader用法01
for imgs, labels in dataloader:
print(imgs.shape)
new_img = torch.squeeze(imgs)
tensorToimg(new_img)
print(labels)
print("dataloader other usages")
#dataloader用法02
for index, (imgs, labels) in enumerate(dataloader):
print(index)
print(imgs.shape)
new_img = torch.squeeze(imgs)
tensorToimg(new_img)
print(labels)