1、from torch.utils.data import DataLoader
主要对数据集做初步划分：可以对数据集进行打乱，每次处理多少batch_size的数据等操作
def __init__(self, dataset, batch_size=1, shuffle=False, sampler=None,
batch_sampler=None, num_workers=0, collate_fn=None,
pin_memory=False, drop_last=False, timeout=0,
worker_init_fn=None, multiprocessing_context=None):
其中几个常用的参数
- dataset 数据集，map-style and iterable-style 可以用index取值的对象、
- batch_size 大小
- shuffle 取batch是否随机取， 默认为False
- sampler 定义取batch的方法，是一个迭代器， 每次生成一个key 用于读取dataset中的值
- batch_sampler 也是一个迭代器， 每次生次一个batch_size的key
- num_workers 参与工作的线程数
- collate_fn 对取出的batch进行处理
- drop_last 对最后不足batchsize的数据的处理方法
其中dataset参数是自定义的数据集处理，它主要返回送入模型训练或测试的数据及其对应的label。它主要加载所有数据，通过Dataloderd进行划分。
LoadDataset继承Dataset，需要重写 len 方法，该方法提供了dataset的大小； getitem 方法， 该方法支持从 0 到 len(self)的索引
class LoadDataset(Dataset):
def __init__(self,datapath,train=True):
self.filenames = []
# trainpath = DATA_PATH + '/CardDetection/images/'
self.imgpath = 'data/input/CardDetection/images/'
self.labelpath = 'data/input/CardDetection/labels/'
self.train=train
# print(datapath)
if os.path.isfile(datapath):
file=datapath.split('/',5)[5]
self.filenames.append(file.split('.')[0])
else:
for file in os.listdir(datapath):
self.filenames.append(file.split('.')[0])
self.nF = len(self.filenames) # number of image files
def __len__(self):
return len(self.filenames)
def __getitem__(self, item):
img=cv2.imread(self.imgpath+self.filenames[item]+'.jpg')
imgpath=self.imgpath+self.filenames[item]+'.jpg'
# img=cv2.imread('data/input/CardDetection/images/0.jpg')
# print(img.shape)#h,w,channel
h,w=img.shape[:2]
input_size=448
#图像增广
padwh=(max(w,h)-min(w,h))//2
if h>w:
img=np.pad(img,((0,0),(padwh,padwh),(0,0)),'constant',constant_values=0)
elif h<w:
img = np.pad(img, ((padwh, padwh),(0, 0) , (0, 0)), 'constant', constant_values=0)
#调整图像
img=cv2.resize(img,(input_size,input_size))
#
# # cv2.imshow('p',img)
img=torch.from_numpy(img.transpose(2, 0, 1)).float()
# cv2.waitKey()
# print(img)
with open(self.labelpath+self.filenames[item]+".txt") as f:
# with open('data/input/CardDetection/labels/0.txt')as f:
bbox=f.read().split('\n')
bbox=[x.split() for x in bbox]
# 字符转数字,
bbox=[float(x) for y in bbox for x in y]
# print(bbox)
target_box=bbox.copy()
#每个text文件中数据：标签、x,y,w,h
for i in range(len(bbox)//5):
# x1, y1 = (int(bbox[i*5+1] * w - bbox[i*5+3] * w / 2), int(bbox[i*5+2] * h - bbox[i*5+4] * h / 2)) # 左上
# x2, y2 = (int(bbox[i*5+1] * w + bbox[i*5+3] * w / 2), int(bbox[i*5+2] * h + bbox[i*5+4] * h / 2)) # 右下
# print(x1, y1, x2, y2)
# # ascontiguousarray函数将一个内存不连续存储的数组转换为内存连续存储的数组，使得运行速度更快
# cv2.rectangle(img, (x1, y1), (x2, y2), color=(255, 255, 0), thickness=3)
# cv2.putText(img, classes[int(bbox[i*5])], (x1, y1), cv2.FONT_HERSHEY_SIMPLEX, 2, (0, 0, 255), thickness=3)
# cv2.imwrite('./testpic/' + "origin" + ".jpg", img)
# cv2.imshow('pic', img)
# cv2.waitKey()
#增广后(max(w,h),max(w,h))微调x,y,w,h
if h>w:
bbox[i * 5 + 1] = (bbox[i * 5 + 1] * w+ padwh) / h
bbox[i * 5 + 3] = (bbox[i * 5 + 3] * w) / h
elif w>h:
bbox[i * 5 + 2] = (bbox[i * 5 + 2] * h+ padwh) / w
bbox[i * 5 + 4] = (bbox[i * 5 + 4] * h) / w
# #
# x1, y1 = (int(bbox[i*5+1] * max(w,h) - bbox[i*5+3] * max(w,h) / 2), int(bbox[i*5+2] * max(w,h) - bbox[i*5+4] * max(w,h) / 2)) # 左上
# x2, y2 = (int(bbox[i*5+1] * max(w,h) + bbox[i*5+3] * max(w,h) / 2), int(bbox[i*5+2] * max(w,h) + bbox[i*5+4] * max(w,h) / 2)) # 右下
# print(x1, y1, x2, y2)
# # ascontiguousarray函数将一个内存不连续存储的数组转换为内存连续存储的数组，使得运行速度更快
# cv2.rectangle(img, (x1, y1), (x2, y2), color=(255, 255, 0), thickness=2)
# cv2.putText(img, classes[int(bbox[i*5])], (x1, y1), cv2.FONT_HERSHEY_SIMPLEX, 2, (0, 0, 255), thickness=2)
# cv2.imwrite('./testpic/' + "pading" + ".jpg", img)
# cv2.imshow('pic', img)
# cv2.waitKey()
# x1, y1 = (int(bbox[i*5+1] * input_size - bbox[i*5+3] * input_size / 2), int(bbox[i*5+2] * input_size - bbox[i*5+4] * input_size / 2)) # 左上
# x2, y2 = (int(bbox[i*5+1] * input_size + bbox[i*5+3] * input_size / 2), int(bbox[i*5+2] * input_size + bbox[i*5+4] * input_size / 2)) # 右下
# print(x1, y1, x2, y2)
# # ascontiguousarray函数将一个内存不连续存储的数组转换为内存连续存储的数组，使得运行速度更快
# cv2.rectangle(img, (x1, y1), (x2, y2), color=(255, 255, 0), thickness=1)
# cv2.putText(img, classes[int(bbox[i*5])], (x1, y1), cv2.FONT_HERSHEY_SIMPLEX, 2, (0, 0, 255), thickness=1)
#
#
# cv2.imwrite('./testpic/' + 'resize' + ".jpg", img)
# cv2.imshow('pic', img)
# cv2.waitKey()
#
# [ 7, 7,34]
if self.train:
labels = bbox2labels(bbox)#数组
#torch.Size([34, 7, 7])
labels=transform.ToTensor()(labels)
else:
labels=torch.tensor(target_box)
# print("labels.shape:", labels.shape)
# print("img.shape:", img.shape)
return imgpath,img,labels②直接编写LoadImagesAndLabels类，传入数据集：img、label的路径，与Dataloderd区别是在内部自己实现按批次数据进行存储，并可以shuffle。
相同点都需要重新编写类内部的固有属性：__iter__、__next__(self)，__len__(self)
class LoadImagesAndLabels: # for training
def __init__(self, path, batch_size=1, img_size=608, augment=False):
with open(path, 'r') as file:
self.img_files = file.readlines()
self.img_files = [x.replace('\n', '') for x in self.img_files]
#图片所有路径
self.img_files = list(filter(lambda x: len(x) > 0, self.img_files))
#存放标签
self.label_files = [x.replace('images', 'labels').replace('.png', '.txt').replace('.jpg', '.txt')
for x in self.img_files]
self.nF = len(self.img_files) # number of image files
self.nB = math.ceil(self.nF / batch_size) # number of batches
self.batch_size = batch_size
self.img_size = img_size
self.augment = augment
assert self.nF > 0, 'No images found in %s' % path
def __iter__(self):
self.count = -1
self.shuffled_vector = np.random.permutation(self.nF) if self.augment else np.arange(self.nF)
return self
def __next__(self):
self.count += 1
if self.count == self.nB:
raise StopIteration
ia = self.count * self.batch_size #以batch_size为步长的训练集位置
ib = min((self.count + 1) * self.batch_size, self.nF)#以batch_size为步长的标签位置
img_all, labels_all, img_paths, img_shapes = [], [], [], []
for index, files_index in enumerate(range(ia, ib)):
img_path = self.img_files[self.shuffled_vector[files_index]]
label_path = self.label_files[self.shuffled_vector[files_index]]
img = cv2.imread(img_path) # BGR
assert img is not None, 'File Not Found ' + img_path
augment_hsv = True
if self.augment and augment_hsv:
# SV augmentation by 50%
fraction = 0.50
#颜色转换hsv处理后再转换
img_hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
S = img_hsv[:, :, 1].astype(np.float32)
V = img_hsv[:, :, 2].astype(np.float32)
a = (random.random() * 2 - 1) * fraction + 1
S *= a
if a > 1:
np.clip(S, a_min=0, a_max=255, out=S)
a = (random.random() * 2 - 1) * fraction + 1
V *= a
if a > 1:
np.clip(V, a_min=0, a_max=255, out=V)
img_hsv[:, :, 1] = S.astype(np.uint8)
img_hsv[:, :, 2] = V.astype(np.uint8)
cv2.cvtColor(img_hsv, cv2.COLOR_HSV2BGR, dst=img)
h, w, _ = img.shape
#调整图像尺寸——>416
img, ratio, padw, padh = letterbox(img, height=self.img_size)
# Load labels
if os.path.isfile(label_path):
labels0 = np.loadtxt(label_path, dtype=np.float32).reshape(-1, 5)
# Normalized xywh to pixel xyxy format
#已经归一化坐标的label中whxy转化为在新尺寸下的非归一化的左上右下坐标
labels = labels0.copy()
#新尺寸/原尺寸 *原尺寸*(归一化的中心坐标-宽高/2)+宽高的补偿
labels[:, 1] = ratio * w * (labels0[:, 1] - labels0[:, 3] / 2) + padw #x1-w/2
labels[:, 2] = ratio * h * (labels0[:, 2] - labels0[:, 4] / 2) + padh#y1-h/2
labels[:, 3] = ratio * w * (labels0[:, 1] + labels0[:, 3] / 2) + padw#x2-w/2
labels[:, 4] = ratio * h * (labels0[:, 2] + labels0[:, 4] / 2) + padh#y2-h/2
else:
labels = np.array([])
# Augment image and labels
if self.augment:
img, labels, M = random_affine(img, labels, degrees=(-5, 5), translate=(0.10, 0.10), scale=(0.90, 1.10))
plotFlag = False
if plotFlag:
import matplotlib.pyplot as plt
plt.figure(figsize=(10, 10)) if index == 0 else None
plt.subplot(4, 4, index + 1).imshow(img[:, :, ::-1])
plt.plot(labels[:, [1, 3, 3, 1, 1]].T, labels[:, [2, 2, 4, 4, 2]].T, '.-')
plt.axis('off')
nL = len(labels)
if nL > 0:
# convert xyxy to xywh
#将新获得的xyxy坐标转化为中心坐标，在除以新图的尺寸 （归一化）
labels[:, 1:5] = xyxy2xywh(labels[:, 1:5].copy()) / self.img_size
if self.augment:
# random left-right flip
#左右旋转
lr_flip = True
if lr_flip & (random.random() > 0.5):
img = np.fliplr(img)
if nL > 0:
labels[:, 1] = 1 - labels[:, 1]
# random up-down flip
#上下旋转
ud_flip = False
if ud_flip & (random.random() > 0.5):
img = np.flipud(img)
if nL > 0:
labels[:, 2] = 1 - labels[:, 2]
if nL > 0:
labels = np.concatenate((np.zeros((nL, 1), dtype='float32') + index, labels), 1)#1维度连接
labels_all.append(labels)
img_all.append(img)
img_paths.append(img_path)
img_shapes.append((h, w))
# Normalize
img_all = np.stack(img_all)[:, :, :, ::-1].transpose(0, 3, 1, 2) # BGR to RGB and cv2 to pytorch
img_all = np.ascontiguousarray(img_all, dtype=np.float32)
img_all /= 255.0
labels_all = torch.from_numpy(np.concatenate(labels_all, 0))
#返回所有图像数据、标签、图像路径、图像尺寸
return torch.from_numpy(img_all), labels_all, img_paths, img_shapes
def __len__(self):
return self.nB # number of batches
2、from torch.utils.data import TensorDataset
TensorDataset 可以用来对 tensor 进行打包，就好像 python 中的 zip 功能。该类通过每一个 tensor 的第一个维度进行索引。因此，该类中的 tensor 第一维度必须相等。（参考）
例如：读取一个csv数据集文件：首先提取特征样本和标签，然后用sklearn中的train_test_split对数据及划分为训练样本及标签，测试样本及标签
涉及到数据格式转化：pandas->numpy->tensor,然后将训练集及对应标签打包使用TensorDataset，在使用DataLoader划分数据集
data=pd.read_csv("irs.csv")
# print(data)
xdata=data.iloc[:,:-1]
ydata=data.iloc[:,-1]
# print(ydata)
#数据预处理：数据离散程度较大-->归一化
#one-hot 编码也是离散特征无意义
#pd转numpy
xdata=xdata.values
# 将非数字标签转数字
class_le=LabelEncoder()
ydata=class_le.fit_transform(ydata.values)
# print(ydata)
# print(xdata)
print()
xtrain,xtest,ytrain,ytest=train_test_split(xdata,ydata,test_size=0.3,shuffle=True)
#Numpy->tensor
xtrain=torch.from_numpy(xtrain).type(torch.float32)
xtest=torch.from_numpy(xtest).type(torch.float32)
ytrain=torch.from_numpy(ytrain).type(torch.float32)
ytest=torch.from_numpy(ytest).type(torch.float32)
X=torch.from_numpy(xdata).type(torch.float32)
Y=torch.from_numpy(ydata).type(torch.float32)
lr=0.0001
batches=16
epochs=1000
datatensor=TensorDataset(xtrain,ytrain)
data_dl=DataLoader(datatensor,batch_size=batches,shuffle=True)3、torchvision.datasets
torchvision.datasets这个包中包含MNIST、FakeData、COCO、LSUN、ImageFolder、DatasetFolder、ImageNet、CIFAR等一些常用的数据集，并且提供了数据集设置的一些重要参数设置，可以通过简单数据集设置来进行数据集的调用。
通过ImageFolder加载自己的数据集（在图像分类识别中经常用到，只需要将同一类的数据集放在同一个文件夹下）
root = './data/dog_cat'
dataset_train = datasets.ImageFolder(root + '/train', transform)
dataset_test = datasets.ImageFolder(root + '/test', transform)
trainloder=DataLoader(dataset_train,batch_size=batchs,shuffle=True)
testloder=DataLoader(dataset_test,batch_size=batchs,shuffle=True)也可以联网下载一些数据集
data_train=datasets.MNIST(root='./data/',transform=transform,train=True,download=True)
data_test=datasets.MNIST(root='./data/',transform=transform,train=False)
dl_train=DataLoader(data_train,batch_size=batch_size,shuffle=True)
dl_test=DataLoader(data_test,batch_size=batch_size,shuffle=False)
参考：
https://pytorch.org/docs/master/data.html#torch.utils.data.SubsetRandomSampler
扫一扫
357
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
