PyTorch训练中Dataset多线程加载数据，而不是在DataLoader

self.batches = self._get_len_batches(self.percentage)

def _get_len_batches(self, percentage):

“”"

Description:

• you could control how many batches you want to use for training or validating

indices sort, so that could keep the batches got in order from originla batches

Parameters:

• percentage: float, range [0, 1]

Return

• numpy array of the new bags

“”"

batch_num = int(len(self.batches) * percentage)

indices = random.sample(list(range(len(self.batches))), batch_num)

indices.sort()

new_batches = np.array(self.batches, dtype=‘object’)[indices]

return new_batches

def _create_batches(self, ):

if self.shuffle:

random.shuffle(self.images)

batches = []

ranges = list(range(0, len(self.images), self.batch_num))

for i in ranges[:-1]:

batch = self.images[i:i + self.batch_num]

batches.append(batch)

== Drop last ===============================================

last_batch = self.images[ranges[-1]:]

if len(last_batch) == self.batch_num:

batches.append(last_batch)

elif self.drop_last:

pass

else:

batches.append(last_batch)

return batches

def getitem(self, index):

batch = self.batches[index]

== Stack all images, become a 4 dimensional tensor ===============

if self.multi_load:

batch_images = self._multi_loader(batch)

else:

batch_images = []

for image in batch:

img = self._load_transform(image)

batch_images.append(img)

batch_images_tensor = torch.stack(batch_images, dim=0)

return batch_images_tensor

def _load_transform(self, tile):

img = self.loader(tile)

if self.transform is not None:

image_np = np.array(img)

augmented = self.transform(image=image_np)

img = augmented[‘image’]

return img

def _multi_loader(self, tiles):

images = []

executor = ThreadPoolExecutor(max_workers=self.num_workers)

results = executor.map(self._load_transform, tiles)

executor.shutdown()

for result in results:

images.append(result)

return images

def