pytorch加载数据主要学习了两种:只有图片的数据集和有scv保存标签的数据集 而第一种只有图片的数据集的加 载又分为两种:标签在文件夹上的和标签在图片名上的
1.第一种标签在文件夹上的数据加载方法
输入:
import os
os.environ['KMP_DUPLICATE_LTB_OK']='True'
#上面的语句是由于我的python与torch版本不匹配才加的与加载数据无关
from torch.utils.data import Dataset,DataLoader
from torchvision import transforms,utils,datasets
import matplotlib.pyplot as plt
from PIL import Image
import numpy as np
import pandas as pd
import torch
import warnings#而这两句则是为了将程序运行时的警告显示去除用的
warnings.filterwarnings("ignore")
”“”
完整程序:
data_transform = transforms.Compose([
transforms.Resize(32), # 缩放图片(Image),保持长宽比不变,最短边为32像素
transforms.CenterCrop(32), # 从图片中间切出32*32的图片
transforms.ToTensor(), # 将图片(Image)转成Tensor,归一化至[0, 1]
transforms.Normalize(mean=[0.492, 0.461, 0.417], std=[0.256, 0.248, 0.251])
# 标准化至[-1, 1],规定均值和标准差
])
hymenoptera_dataset = datasets.ImageFolder\
(root="F:\\jupyter\\pytorch\\data\\dogs_cats\\dataset\\train",
transform=data_transform)
dataset_loader = torch.utils.data.DataLoader(hymenoptera_dataset,batch_size=4,shuffle=True)
“”“
data_transform=transforms.Compose([
transforms.Resize(32), # 缩放图片(Image),保持长宽比不变,最短边为32像素
transforms.CenterCrop(32), # 从图片中间切出32*32的图片
transforms.ToTensor(), # 将图片(Image)转成Tensor,归一化至[0, 1]
transforms.Normalize(mean=[0.492, 0.461, 0.417], std=[0.256, 0.248, 0.251])
# 标准化至[-1, 1],规定均值和标准差
])
"""
此函数的作用是对图片进行标准化和归一化 Resize(32)缩放图片(Image),保持长宽比不变,
最短边为32像素 CenterCrop(32)从图片中间切出32*32的图片
RandomSizedCrop(32)这一句的作用是对原图进行随机大小和高宽比的裁剪,最后的尺寸为32x32
RandomHorizontalFlip()这个则是对原图像根据概率进行随机水平翻转
transforms.ToTensor()将图片转化为张量,并使图片的形式表现为通道x高x宽的形式
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
这个则是对数据 进行正则化操作,第一个参数为均值,第二个参数为标准差。这两个值我是使用程序进行计算的。
代码如下
"""
import cv2
import os
img_h,img_w=32,32
means,stdevs=[],[]
img_list=[]
imgs_path=r"D:\caocao\hotdog\train"
imgs_path_list=os.listdir(imgs_path)
len_=len(imgs_path_list)
for item in imgs_path_list:
img=cv2.imread(os.path.join(imgs_path,item))
imgs=cv2.resize(img,(img_w,img_h))
img=img[:,:,:,np.newaxis]
img_list.append(img)
i=i+1
print(i,'/',len_)
imgs=np.concatenate(img_list,axis=3)
imgs=imgs.astype(np.float32)/255.
for i in range(3):
pixels=imgs[:,:,i,:].ravel()
means.append(np.mean(pixels))
stdevs.append(np.std(pixels))
means.reverse()
stdevs.reverse()
print("normMean = {}".format(means))
print("normStd = {}".format(stdevs))
hymenoptera_dataset=datasets.ImageFolder(root="D:\caocao\hotdog\train",
transforms=data_transform)
img,label=hymenoptera_dataset[0]
print(label)
print(img.size)
for img,label in hymenoptera_dataset:
print("图像img的形状{},标签label的值{}".format(img.shape, label))
print("图像数据预处理后:\n", img)
break
dataset_loader=torch.utils.data.DataLoader(hymenoptera_dataset,batch_size=4,shuffle=True)
import torchvision
import matplotlib.pyplot as plt
import numpy as np
%matplotlib inline
# 显示图像
def imshow(img):
img=img/2+0.5
npimg=img.numpy()
plt.imshow(np.transpose(npimg,(1,2,0)))
plt.show()
dataiter=iter(dataset_loader)
images,labels=dataiter.next()
imshow(torchvision.utils.make_grid(images))
print(' '.join('%s' %["小狗" if labels[j].item()==1 else "小猫" for j in range(4)]))
2. 第二种标签在图片名上的数据加载方法
import os
os.environ['KMP_DUPLICATE_LTB_OK']='True'
#上面的语句是由于我的python与torch版本不匹配才加的与加载数据无关
from torch.utils.data import Dataset,DataLoader
from torchvision import transforms,utils,datasets
import matplotlib.pyplot as plt
from PIL import Image
import numpy as np
import pandas as pd
import torch
import warnings#而这两句则是为了将程序运行时的警告显示去除用的
warnings.filterwarnings("ignore")
class MyDataset(Dataset):
def __init__(self,path_dir,transform=None):
self.path_dir=path_dir
self.transform=transform
self.images=os.listdir(self.path_dir)
def __len__(self):
return len(self.images)
def __getitem__(self,index):
image_index=self.images[index]
img_path=os.path.join(self.path_dir,image_index)
Image.open(img_path).convert("RGB")
label=image_path.split('\\'')[-1].split('.')[0]
label=1 if 'dog' in label else 0
if self.transform is not None:
img=self.transform(img)
return img,label
from torchvision import transforms as T
transform = T.Compose([
T.Resize(32), # 缩放图片(Image),保持长宽比不变,最短边为224像素
T.CenterCrop(32), # 从图片中间切出224*224的图片
T.ToTensor(), # 将图片(Image)转成Tensor,归一化至[0, 1]
T.Normalize(mean=[.5, .5, .5], std=[.5, .5, .5]) # 标准化至[-1, 1],规定均值和标准差
])
dataset = MyDataset("F:\\jupyter\\pytorch\\data\\dataset_kaggledogvscat\\data\\train", transform=transform)
dataloader = DataLoader(dataset, batch_size=4, shuffle=True)
4.csv文件存储标签的数据集的加载的方法
输入:
#数据集的分割
import math
import os
import shutil
from collections import Counter
def reorg_dog_data(data_dir, label_file, train_dir, test_dir, input_dir,
valid_ratio):
# 读取训练数据标签。
with open(os.path.join(data_dir, label_file), 'r') as f:
# 跳过文件头行(栏名称)。
lines = f.readlines()[1:]
tokens = [l.rstrip().split(',') for l in lines]
idx_label = dict(((idx, label) for idx, label in tokens))
labels = set(idx_label.values())
num_train = len(os.listdir(os.path.join(data_dir, train_dir)))
# 训练集中数量最少一类的狗的数量。
min_num_train_per_label = (
Counter(idx_label.values()).most_common()[:-2:-1][0][1])
# 验证集中每类狗的数量。
num_valid_per_label = math.floor(min_num_train_per_label * valid_ratio)
label_count = dict()
def mkdir_if_not_exist(path):
if not os.path.exists(os.path.join(*path)):
os.makedirs(os.path.join(*path))
# 整理训练和验证集。
for train_file in os.listdir(os.path.join(data_dir, train_dir)):
idx = train_file.split('.')[0]
label = idx_label[idx]
mkdir_if_not_exist([data_dir, input_dir, 'train_valid', label])
shutil.copy(os.path.join(data_dir, train_dir, train_file),
os.path.join(data_dir, input_dir, 'train_valid', label))
if label not in label_count or label_count[label] < num_valid_per_label:
mkdir_if_not_exist([data_dir, input_dir, 'valid', label])
shutil.copy(os.path.join(data_dir, train_dir, train_file),
os.path.join(data_dir, input_dir, 'valid', label))
label_count[label] = label_count.get(label, 0) + 1
else:
mkdir_if_not_exist([data_dir, input_dir, 'train', label])
shutil.copy(os.path.join(data_dir, train_dir, train_file),
os.path.join(data_dir, input_dir, 'train', label))
# 整理测试集。
mkdir_if_not_exist([data_dir, input_dir, 'test', 'unknown'])
for test_file in os.listdir(os.path.join(data_dir, test_dir)):
shutil.copy(os.path.join(data_dir, test_dir, test_file),
os.path.join(data_dir, input_dir, 'test', 'unknown'))
reorg_dog_data(data_dir, label_file, train_dir, test_dir, input_dir,
valid_ratio)
#加载数据集
import os
os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
from torch.utils.data import DataLoader,Dataset
import matplotlib.pyplot as plt
import torch
from torchvision import transforms, datasets
from PIL import Image
import pandas as pd
import numpy as np
#过滤警告信息
import warnings
warnings.filterwarnings("ignore")
transform_train = transforms.Compose([
# 随机对图像裁剪出面积为原图像面积0.08~1倍、且高和宽之比在3/4~4/3的图像,再放缩为高和宽均为224像素的新图像
transforms.RandomResizedCrop(28, scale=(0.08, 1.0),
ratio=(3.0/4.0, 4.0/3.0)),
# 以0.5的概率随机水平翻转
transforms.RandomHorizontalFlip(),
# 随机更改亮度、对比度和饱和度
transforms.ColorJitter(brightness=0.4, contrast=0.4, saturation=0.4),
transforms.ToTensor(),
# 对各个通道做标准化,(0.485, 0.456, 0.406)和(0.229, 0.224, 0.225)是在ImageNet上计算得的各通道均值与方差
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) # ImageNet上的均值和方差
])
# 在测试集上的图像增强只做确定性的操作
transform_test = transforms.Compose([
transforms.Resize(28),
# 将图像中央的高和宽均为224的正方形区域裁剪出来
transforms.CenterCrop(28),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])
import torchvision
#然后开始加载数据集
new_data_dir="F:\\jupyter\\pytorch\\data\\dog-breed-identification\\train_valid_test"
train_ds = datasets.ImageFolder(root=os.path.join(new_data_dir, 'train'),transform=transform_train)
valid_ds = datasets.ImageFolder(root=os.path.join(new_data_dir, 'valid'),transform=transform_train)
train_loader = torch.utils.data.DataLoader(train_ds,batch_size=4,shuffle=True)
valid_loader = torch.utils.data.DataLoader(valid_ds,batch_size=4,shuffle=True)
3.
import os
os.environ['KMP_DUPLICATE_LTB_OK']='True'
#上面的语句是由于我的python与torch版本不匹配才加的与加载数据无关
from torch.utils.data import Dataset,DataLoader
from torchvision import transforms,utils,datasets
import matplotlib.pyplot as plt
from PIL import Image
import numpy as np
import pandas as pd
import torch
import warnings#而这两句则是为了将程序运行时的警告显示去除用的
warnings.filterwarnings("ignore")
class MyDataset(Dataset):
def __init__(self,root,transforms=None,train=True,test=False):
self.test=test
imgs=[os.path.join(root,img) for img in os.listdir(root)]
if self.test:
imgs=sorted(imgs,key=lambda x:int(x.split('.')[-2].split('\\')[-1]))
else:
imgs=sorted(imgs,key=lambda x:int(x.split('.')[-2]))
imgs_num=len(imgs)
if self.test:
self.imgs=imgs
elif train:
self.imgs=imgs[:int(0.7*imgs_num)]
else:
self.imgs=imgs[int(0.7*imgs_num):]
if transform is None:
normalize=T.Normalize(mean=[0.488,0.455,0.417])
if self.test or not train:
self.transforms=T.Compose([
T.Resize(28),
T.CenterCrop(28),
T.ToTensor(),
normalize
])
else:
self.transforms= T.Compose([
T.Resize(28),
T.CenterCrop(28),
T.RandomHorizontalFlip(),
T.ToTensor(),
normalize
])
def __len__(self):
return len(self.imgs)
def __getitem__(self, index):
img_path=self.imgs[index]
if self.test:
label=int(self.imgs[index].split('.')[-2].split('\\')[-1])
else:
label=1 if 'dog' in img_path.split('\\')[-1] else 0
data=Image.open(img_path)
data=self.transforms(img)
return data,label
train_data = MyDataset("F:\\jupyter\\pytorch\\data\\dataset_kaggledogvscat\\data\\train",
train=True)
val_data = MyDataset("F:\\jupyter\\pytorch\\data\\dataset_kaggledogvscat\\data\\train",
train=False)
'''
训练集train_data和验证集val_data加载数据时,需要将第二个参数train设置为对应的值,便可以获取对应的数据
由于dataset中直接默认进行了transforms操作所以如果需要获取未进行数据加强操作的dataseet
需要对transforms相关的操作进行注释掉即可。
#使用DataLoader加载训练集数据
train_dataloader = DataLoader(train_data,batch_size=4,shuffle=True)
for batch_datas, batch_labels in dataloader:
print(batch_datas.size(),batch_labels.size())
break
#使用DataLoader加载验证集数据
val_dataloader = DataLoader(val_data,batch_size=4,shuffle=True)
for batch_datas, batch_labels in dataloader:
print(batch_datas.size(),batch_labels.size())
break
'''
示例1:G:\deep-learn\deep-learning\代码总结\日月光华 数据集加载.py
输入:
image=glob.glob(r'dataset/images/*.jpg')
annotations=glob.glob(r'dataset/annotations/trimaps/*.png')
np.random.seed(2022)
index=np.random.permutation(len(images))
images=np.array(images)[index]
annotations=np.array(annotation)[index]
sep=int(len(images)*0.8)
train_imgs=images[:sep]
train_annos=annotations[:sep]
test_imgs=images[sep:]
test_annos=annotations[sep:]
#annotation 像素级 0,1,2,3
完整代码:
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils import data
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
import torchvision
from torchvision import transforms
import os
import glob
from PIL import Image
image=glob.glob(r'dataset/images/*.jpg')
annotations=glob.glob(r'dataset/annotations/trimaps/*.png')
np.random.seed(2022)
index=np.random.permutation(len(images))
images=np.array(images)[index]
annotations=np.array(annotation)[index]
sep=int(len(images)*0.8)
train_imgs=images[:sep]
train_annos=annotations[:sep]
test_imgs=images[sep:]
test_annos=annotations[sep:]
#annotation 像素级 0,1,2,3
transform = transforms.Compose([
transforms.Resize((256, 256)),
transforms.ToTensor(),
])
class Oxford_dataset(data.Dataset):
def __init__(self,img_path,anno_path):
self.imgs=img_path
self.annos=anno_path
def __getitem__(self, item):
img=self.imgs[item]
anno=self.annos[item]
pil_img=Image.open(img).convert('RGB')
pil_tensor=transform(pil_img)
pil_anno=Image.open(anno).resize((256,256))
anno_tensor=torch.tensor(np.array(pil_anno),dtype=torch.int64)
return img_tensor,anno_tensor-1 #1,2,3->0,1,2
def __len__(self):
return len(self.imgs)
train_dataset=Oxford_dataset(train_imgs,train_annos)
test_dataset=Oxford_dataset(test_imgs,test_annos)
batch_size=8
train_dl=data.DataLoader(train_dataset,batch_size=batch_size,
shuffle=True)
test_dl=data.DataLoader(test_dataset,batch_size=batch_size,
)
img_batch,anno_batch=next(iter(train_dl))
'''''''''''''''''''''文本分类'''''''''''''''''
train_iter,test_iter=torchtext.datasets.IMDB()
next(train_iter)
('neg',
'I am ...')
#neg 标签,其他是内容
train_data,test_data=list(train_iter),list(test_iter)
all_classes=set([label for (label,text) in train_data])
#{'neg','pos'}
num_class=len(all_classes)
#导入分词工具
from torchtext.data.utils import get_tokenizer
tokenier=get_tokenizer('basic_english')
tokenier('This is a riyueguanghua Pytorch lesson')
#['this','is', 'a', 'riyueguanghua', 'pytorch', 'lesson']
from torchtext.vocab import build_vocab_from_iterator
#定义生成器,返回以分词的文本
def yield_tokens(data):
for (_,text) in data:
yield tokenier(text)
vocab=build_vocab_from_iterator(yield_tokens(train_data),
specials=['<pad>','<unk>'],
min_freq=3)
vocab['look']
#‘look 编码174
vocab['film']
# ‘film' 编码23
vocab.set_default_index(vocab['<unk>'])#不认识的单词默认编码为1
vocab['riyueguanghua']
#1
vocab(['the','file'])
#[2,7778]
inp='this is a riyue'
vocab(tokenier(inp))
#[14,10,6,1]
text_pipeline=lambda x:vocab(tokenier(x))
text_pipeline('this is a riyue')
#[14,10,6,1]
label_pipeline=lambda x:int(x=='pos')
label_pipeline('neg')
#0
label_pipeline('pos')
#1
from torch.utils.data import DataLoader
text_dl=DataLoader(train_data,
batch_size=64,
shuffle)
def collate_batch(text_label_batch):
# text_label_batch返回一个元组,两个值
# text_label_batch=next(train_iter)
# ('neg',
# 'I am ...',
# )
# # neg 标签,其他是内容。pos是标签,
label_list,test_list,offset_list=[],[],[0]#第一个偏移量为0
#offset_list保存每一条评论的长度,offset代表偏移值,
#tensor(1,2,3,6,7,7,9,0,0,0),offset:0,,4,5>>>,每一条评论的起始位置
for _label,_text in text_label_batch:
label_list.append(label_pipeline(_label))
precess_text=torch.tensor(text_pipeline(_text),
dtype=torch.int64)
text_list.append(precess_text)
offset_list.append(precess_text.size(0))#0维
label_list=torch.tensor(label_list)
text_list=torch.cat(text_list)
offsets=torch.tensor(offset_list[:-1]).cumsum(dim=0)
return label_list.to(device),text_list.to(device),offsets
train_dl=data.DataLoader(train_dataset,batch_size=batch_size,
shuffle=True,
collate_fn=collate_batch())
test_dl=data.DataLoader(test_dataset,batch_size=batch_size,
collate_fn=collate_batch())