1. 数据集下载
https://www.kaggle.com/c/dog-breed-identification
2. 目录结构
3. config.py
FILE_PATH = './dog-breed-identification'
TEST_PATH = './dog-breed-identification/test'
TRAIN_PATH = './dog-breed-identification/train'
CSV_PATH = './dog-breed-identification/labels.csv'
4. dataloader.py
import pandas as pd
import dog_loader.config as conf
from pandas import Series, DataFrame
import numpy as np
import os
import PIL
from torch.utils.data import Dataset, DataLoader
from torchvision import transforms, utils
# 一. 查看表格信息
df = pd.read_csv(conf.CSV_PATH)
print(df.info())
print(df.head())
print('*' * 60)
# 二.预处理
# 1)得到一个长 list1 : 里面是每张图片的路径
# 2)另外一个长list2: 里面是每张图片对应的标签(整数),顺序要和list1对应。
# 3)把这两个list切分出来一部分作为验证集
breed = df['breed']
print(type(breed))
breed_np = breed.values
print(type(breed_np))
print(breed_np.shape)
# 看一下一共多少不同种类 #120
breed_set = set(breed_np)
print('breed_set_size', len(breed_set))
# 构建一个编号与名称对应的字典,以后输出的数字要变成名字的时候用:
breed_120_list = list(breed_set)
dic = {}
for i in range(120):
dic[breed_120_list[i]] = i
print('*' * 60)
# 处理id那一列,分割成两段:
file = df["id"].values
print('file.shape', file.shape)
file = [i + ".jpg" for i in file]
file = [os.path.join(conf.TRAIN_PATH, i) for i in file]
file_train = file[:8000]
file_val = file[8000:]
# print(file_train)
np.save("file_train.npy", file_train)
np.save("file_val.npy", file_val)
# 处理breed那一列,分成两段:
breed = df["breed"].values
print('breed.shape', breed.shape)
number = []
for i in range(10222):
number.append(dic[breed[i]])
number = np.array(number)
number_train = number[:8000]
number_val = number[8000:]
np.save("number_train.npy", number_train)
np.save("number_val.npy", number_val)
normalize = transforms.Normalize(
mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]
)
preprocess = transforms.Compose([
# transforms.Scale(256),
# transforms.CenterCrop(224),
transforms.ToTensor(),
normalize
])
def default_loader(path):
img_pil = PIL.Image.open(path)
img_pil = img_pil.resize((224, 224))
img_tensor = preprocess(img_pil)
return img_tensor
# 当然出来的时候已经全都变成了tensor
class trainset(Dataset):
def __init__(self, loader=default_loader):
# 定义好 image 的路径
self.images = file_train
self.target = number_train
self.loader = loader
def __getitem__(self, index):
fn = self.images[index]
img = self.loader(fn)
target = self.target[index]
return img, target
def __len__(self):
return len(self.images)
# 自定义Dataset只需要最下面一个class,继承自Dataset类。有三个私有函数
# def __init__(self, loader=default_loader):
# 这个里面一般要初始化一个loader(代码见上面),一个images_path的列表,一个target的列表
# def __getitem__(self, index):
# 这里吗就是在给你一个index的时候,你返回一个图片的tensor和target的tensor,使用了loader方法,经过 归一化,剪裁,类型转化,从图像变成tensor
# def __len__(self):
# return 你所有数据的个数
5. dataset.py, 测试dataloader:
import dog_loader.dataloader as dat
from torch.utils.data import Dataset, DataLoader
train_data = dat.trainset()
trainloader = DataLoader(train_data, batch_size=128, shuffle=True)
print('type(trainloader)', type(trainloader))
# 使用enumerate读取
# for index, data in enumerate(trainloader):
# image, label = data
# print(image.size(), label.size())
# print('image_tensor', image[0][0][0][:5])
# print('label', label[:5])
for index, (image, label) in enumerate(trainloader):
print(image.size(), label.size())
# print('image_tensor', image[0][0][0][:5])
# print('label', label[:5])
print('-'*25, index, '-'*25)
6. 参考: