pytorch加载自己的数据集

1.数据集格式

X_train,X_test都是二进制文件pickle形式
shape为 (10240,64,8)类型
8分类问题

2.模仿Mnist加载方式

batch_size = 64
transform = transforms.Compose([transforms.ToTensor(),
                                transforms.Normalize((0.1307,), (0.3081,))])
""" 
一定要有dataset和DataLoader才行
"""
train_dataset = datasets.MNIST(root='../dataset/mnist/',
                               train=True, download=True,
                               transform=transform)

train_loader = DataLoader(dataset=train_dataset,
                          shuffle=True,
                          batch_size=batch_size)

要学会查看源码，看一下datasets.Mnist( ) 内部的代码
如下所示，里面有多个参数，我们比较关心transform，可以看看__getitem__里，做了什么处理

    def __init__(self, root, train=True, transform=None, target_transform=None, download=False):
        self.root = os.path.expanduser(root)
        self.transform = transform
        self.target_transform = target_transform
        self.train = train  # training set or test set

这里的__getitem()__，先用index取出单个样本和标签，
再转成PIL图像的格式
再经过transform转成Tensor或者同时有normalize处理
最后return 就行

    def __getitem__(self, index):
        """
        Args:
            index (int): Index

        Returns:
            tuple: (image, target) where target is index of the target class.
        """
        if self.train:
            img, target = self.train_data[index], self.train_labels[index]
        else:
            img, target = self.test_data[index], self.test_labels[index]

        # doing this so that it is consistent with all other datasets
        # to return a PIL Image
        img = Image.fromarray(img.numpy(), mode='L')

        if self.transform is not None:
            img = self.transform(img)

        if self.target_transform is not None:
            target = self.target_transform(target)

        return img, target

3.Normalize

transforms.Normalize((0.1307,), (0.3081,))])

单通道，所以是mean和方差 只有一个

另外，对于均值和方差，是所有像素点加起来计算的

4.普通单通道计算均值、方差

最简单的方式就是

"""(10240,64,8)"""
def SS_pro(data):
    size_0 = data.shape[0]
    size_1 = data.shape[1]
    """先拉平，因为SS只能处理两维的"""
    data = data.reshape(size_0,-1)
    x = StandardScaler().fit_transform(data)
    x = x.reshape(size_0,size_1,-1)
    return x

"""min_max处理，看哪个效果好  feature_range控制输出范围"""
def MM_pro(data):
    size_0 = data.shape[0]
    size_1 = data.shape[1]
    data = data.reshape(size_0, -1)
    """feature_range控制范围"""
    x = MinMaxScaler(feature_range=(0, 1)).fit_transform(data)
    x = x.reshape(size_0, size_1, -1)
    return x

4.2.稍微复杂点的方式

此小节参考来源

import numpy as np
import cv2
import random
 
# calculate means and std
train_txt_path = './train_val_list.txt'
 
CNum = 10000     # 挑选多少图片进行计算
 
img_h, img_w = 32, 32
"""imgs是后面要拼接起来的，格式为w,h,c,batch
batch=1,意思就是单张拼接，依次
"""
imgs = np.zeros([img_w, img_h, 3, 1])
"""means是"""
means, stdevs = [], []
 
with open(train_txt_path, 'r') as f:
    lines = f.readlines()
    random.shuffle(lines)   # shuffle , 随机挑选图片
 
    for i in tqdm_notebook(range(CNum)):
        img_path = os.path.join('./train', lines[i].rstrip().split()[0])
 
        img = cv2.imread(img_path)
        img = cv2.resize(img, (img_h, img_w))
        img = img[:, :, :, np.newaxis]
        """按第3轴拼接，就是图片数量叠加"""
        imgs = np.concatenate((imgs, img), axis=3)
#         print(i)
"""先除以255，归一化"""
imgs = imgs.astype(np.float32)/255.
 
 
for i in tqdm_notebook(range(3)):
"""把单通道所有数据取出来，拉平，再求均值和方差"""
    pixels = imgs[:,:,i,:].ravel()  # 拉成一行
    means.append(np.mean(pixels))
    stdevs.append(np.std(pixels))
 
# cv2 读取的图像格式为BGR，PIL/Skimage读取到的都是RGB不用转
means.reverse() # BGR --> RGB
stdevs.reverse()
 
print("normMean = {}".format(means))
print("normStd = {}".format(stdevs))
"""这里有transform归一化"""
print('transforms.Normalize(normMean = {}, normStd = {})'.format(means, stdevs))

4.3如果自己是三通道矩阵

只需要这一段就行，而且自己改一下
把单个通道的数据都取出来

for i in tqdm_notebook(range(3)):
"""把单通道所有数据取出来，拉平，再求均值和方差"""
    pixels = imgs[:,:,i,:].ravel()  # 拉成一行
    means.append(np.mean(pixels))
    stdevs.append(np.std(pixels))
 
# cv2 读取的图像格式为BGR，PIL/Skimage读取到的都是RGB不用转
means.reverse() # BGR --> RGB
stdevs.reverse()

当然，更简单的方式是利用StandardScaler缩放一下，

5.自己定义dataset后传到dataloader

class Train_set(Dataset):
    def __init__(self, filepath1, filepath2, transform=None):
        """filepath1代表x_train的   filepath2代表y_train的路径"""
        x = getdata(filepath1)
        x = SS_pro(x)
        self.len = x.shape[0]
        self.x_data = x

        y = getdata(filepath2)
        self.y_data = y

        self.transform = transform

    def __getitem__(self, index):
    
        x_data = self.x_data[index]
        target = self.y_data[index]
        
        img = Image.fromarray(x_data, mode='L')
        if self.transform is not None:
            img = self.transform(img)
        
        return img, target

    def __len__(self):
        return self.len


transform = transforms.Compose([transforms.ToTensor()])

fp1 = r'C:\Users\Wu yuhao\Desktop\X_train.p'
fp2 = r'C:\Users\Wu yuhao\Desktop\y_train.p'

dataset = Train_set(fp1, fp2, transform)
train_loader = DataLoader(dataset=dataset,
                          batch_size=256,
                          shuffle=True)