pytorch kaggle dog-breed-identification 数据集读取 dataloader

1. 数据集下载

https://www.kaggle.com/c/dog-breed-identification

2. 目录结构

3. config.py

FILE_PATH = './dog-breed-identification'
TEST_PATH = './dog-breed-identification/test'
TRAIN_PATH = './dog-breed-identification/train'
CSV_PATH = './dog-breed-identification/labels.csv'

4. dataloader.py

import pandas as pd
import dog_loader.config as conf
from pandas import Series, DataFrame
import numpy as np
import os

import PIL
from torch.utils.data import Dataset, DataLoader
from torchvision import transforms, utils

# 一. 查看表格信息
df = pd.read_csv(conf.CSV_PATH)
print(df.info())
print(df.head())
print('*' * 60)

# 二.预处理
# 1)得到一个长 list1 : 里面是每张图片的路径
# 2)另外一个长list2: 里面是每张图片对应的标签（整数），顺序要和list1对应。
# 3)把这两个list切分出来一部分作为验证集
breed = df['breed']
print(type(breed))
breed_np = breed.values
print(type(breed_np))
print(breed_np.shape)

# 看一下一共多少不同种类 #120
breed_set = set(breed_np)
print('breed_set_size', len(breed_set))

# 构建一个编号与名称对应的字典，以后输出的数字要变成名字的时候用：
breed_120_list = list(breed_set)
dic = {}
for i in range(120):
    dic[breed_120_list[i]] = i
print('*' * 60)

# 处理id那一列，分割成两段：
file = df["id"].values
print('file.shape', file.shape)

file = [i + ".jpg" for i in file]
file = [os.path.join(conf.TRAIN_PATH, i) for i in file]
file_train = file[:8000]
file_val = file[8000:]
# print(file_train)

np.save("file_train.npy", file_train)
np.save("file_val.npy", file_val)

# 处理breed那一列，分成两段：
breed = df["breed"].values
print('breed.shape', breed.shape)
number = []
for i in range(10222):
    number.append(dic[breed[i]])
number = np.array(number)
number_train = number[:8000]
number_val = number[8000:]
np.save("number_train.npy", number_train)
np.save("number_val.npy", number_val)

normalize = transforms.Normalize(
    mean=[0.485, 0.456, 0.406],
    std=[0.229, 0.224, 0.225]
)

preprocess = transforms.Compose([
    # transforms.Scale(256),
    # transforms.CenterCrop(224),
    transforms.ToTensor(),
    normalize
])


def default_loader(path):
    img_pil = PIL.Image.open(path)
    img_pil = img_pil.resize((224, 224))
    img_tensor = preprocess(img_pil)
    return img_tensor


# 当然出来的时候已经全都变成了tensor
class trainset(Dataset):
    def __init__(self, loader=default_loader):
        # 定义好 image 的路径
        self.images = file_train
        self.target = number_train
        self.loader = loader

    def __getitem__(self, index):
        fn = self.images[index]
        img = self.loader(fn)
        target = self.target[index]
        return img, target

    def __len__(self):
        return len(self.images)

# 自定义Dataset只需要最下面一个class,继承自Dataset类。有三个私有函数
# def __init__(self, loader=default_loader):
# 这个里面一般要初始化一个loader(代码见上面),一个images_path的列表，一个target的列表
# def __getitem__(self, index)：
# 这里吗就是在给你一个index的时候，你返回一个图片的tensor和target的tensor,使用了loader方法，经过 归一化，剪裁，类型转化，从图像变成tensor
# def __len__(self):
# return 你所有数据的个数

5. dataset.py, 测试dataloader:

import dog_loader.dataloader as dat
from torch.utils.data import Dataset, DataLoader

train_data = dat.trainset()
trainloader = DataLoader(train_data, batch_size=128, shuffle=True)
print('type(trainloader)', type(trainloader))

# 使用enumerate读取
# for index, data in enumerate(trainloader):
#     image, label = data
#     print(image.size(), label.size())
#     print('image_tensor', image[0][0][0][:5])
#     print('label', label[:5])
for index, (image, label) in enumerate(trainloader):
    print(image.size(), label.size())
    # print('image_tensor', image[0][0][0][:5])
    # print('label', label[:5])
    print('-'*25, index, '-'*25)

6. 参考：

https://zhuanlan.zhihu.com/p/35698470