个人学习,仅供参考
获取训练样本
# -*- coding: utf-8 -*-
import os
import math
import gzip
import pickle
import requests
import numpy as np
url_base = 'http://yann.lecun.com/exdb/mnist_learn/'
key_file = {
'train_img': 'train-images-idx3-ubyte.gz',
'train_label': 'train-labels-idx1-ubyte.gz',
'test_img': 't10k-images-idx3-ubyte.gz',
'test_label': 't10k-labels-idx1-ubyte.gz',
}
dataset_dir = os.path.dirname(os.path.abspath(__file__))
save_file = dataset_dir + '/mnist.pkl'
train_num = 60000
test_num = 10000
img_size = 784
def down_data():
for value in key_file.values():
file_name = dataset_dir + '/' + value
if os.path.exists(file_name):
continue
context = requests.get(url_base + '/' + value)
with open(file_name, 'wb') as f:
f.write(context.content)
def load_data(file_name):
file_path = dataset_dir + '/' + file_name
print('transform ' + file_name + 'to NumPy Array ')
with gzip.open(file_path, 'rb') as f:
# 将缓存中的数据解释为numpy数组,offset设置偏移量
data = np.frombuffer(f.read(), np.uint8, offset=8)
print("Well Done")
return data
def reload_data_pkl():
data = {
'train_img': load_data(key_file['train_img']),
'train_label': load_data(key_file['train_label']),
'test_img': load_data(key_file['test_img']),
'test_label': load_data(key_file['test_label'])
}
with open('mnist.pkl', 'wb') as f:
# 参数-1,选择最新版本的协议,对大型数据进行了存储优化,可以使用4代替
pickle.dump(data, f, -1)
def convert_to_one_hot(Y, C):
"""
:param Y: 索引,Y的大小为6000*1,标记
:param C: 对角矩阵的大小
:return:Y:10*6000
"""
# eye返回对角矩阵
# 将标记矩阵Y返回
Y = np.eye(C)[Y.reshape(-1)].T
return Y
def random_mini_batches(x, y, mini_batch_size=64, seed=0):
np.random.seed(seed)
m = x.shape[1]
mini_batches = []
permutation = lis