使用pytorch手动加载mnist、cifar10数据集

这两天在某个平台上白嫖GPU资源，奈何这个平台不支持用torchvision下载数据集，于是只能用自己电脑下载好数据之后上传再自己手动DIY加载数据集。

参考链接：
加载mnist数据集
加载cifar10数据集

加载mnist数据集并实现简单的全连接层

import numpy as np 
import pandas as pd 
from torchvision import datasets,transforms
from torch.utils.data import DataLoader,Dataset
import torch
import gzip,os

def load_data(data_folder, data_name, label_name):
    with gzip.open(os.path.join(data_folder,label_name), 'rb') as lbpath:
        y_train = np.frombuffer(lbpath.read(), np.uint8, offset=8)
    with gzip.open(os.path.join(data_folder,data_name), 'rb') as imgpath:
        x_train = np.frombuffer(
            imgpath.read(), np.uint8, offset=16).reshape(len(y_train), 28, 28)
    return (x_train, y_train)

class DealDataset(Dataset):
    """
        读取数据、初始化数据
    """
    def __init__(self,folder,data_name,label_name,transform = None):
        (train_set,train_labels) = load_data(folder,data_name,label_name)
        self.train_set = train_set
        self.train_labels = train_labels
        self.transform = transform
    
    def __getitem__(self,index):

        img,target = self.train_set[index],int(self.train_labels[index])
        if self.transform is not None:
            img = self.transform(img)
        return img, target
    
    def __len__(self):
        return len(self.train_set)

加载数据集

数据集事先已经下载好保存在某个文件夹下

trainDataset = DealDataset('../data/MNIST/raw', "train-images-idx3-ubyte.gz","train-labels-idx1-ubyte.gz",transform=transforms.Compose([
                       transforms.ToTensor(),
                       transforms.Normalize((0.1307,), (0.3081,))
                   ]))
testDataset = DealDataset('../data/MNIST/raw', "t10k-images-idx3-ubyte.gz","t10k-labels-idx1-ubyte.gz",transform=transforms.Compose([
        transforms.ToTensor(),
        transforms.Normalize((0.1307,), (0.3081,))
    ]))

定义生成器

batch_size=200
train_loader = DataLoader(
    dataset=trainDataset,
    batch_size=batch_size, # 一个批次可以认为是一个包，每个包中含有100张图片
    shuffle=True,
)

test_loader = DataLoader(
    dataset=testDataset,
    batch_size=batch_size, # 一个批次可以认为是一个包，每个包中含有100张图片
    shuffle=True,)

定义全连接层并计算

import  torch.nn.functional as F
import  torch.optim as optim
import  torch.nn as nn

learning_rate=0.01
epochs=10

class MLP(nn.Module):
    def __init__(self):
        super(MLP,self).__init__()
        
        self.model = nn.Sequential(
        nn.Linear(784,200),
        nn.LeakyReLU(inplace = True),
        nn.Linear(200,200),
        nn.LeakyReLU(inplace = True),
        nn.Linear(200,10),
        nn.LeakyReLU(inplace = True),
        )
    
    def forward(self,x):
        x = self.model(x)
        return x

device = torch.device('cuda:0') 
## 在GPU上计算
net = MLP().to(device)
optimizer = optim.SGD(net.p