python分模块实现神经网络之CIFARF分类：1.加载CIFAR数据集中的数据

加载CIFAR数据集中的数据是在data_utils.py文件中完成的，下面是加载文件的代码：

import numpy as np
import os
#import cPickle as pickle
import pickle as pk 

def load_CIFAR_batch(filename):
  """ load single batch of cifar """
  with open(filename, 'rb') as f:
    datadict = pk.load(f,encoding='latin1')
    X = datadict['data']
    Y = datadict['labels']
    X = X.reshape(10000, 3, 32, 32).transpose(0,2,3,1).astype("float")
    Y = np.array(Y)
    return X, Y

def load_CIFAR10(ROOT):
  """ load all of cifar """
  xs = []
  ys = []
  for b in range(1,2):
    f = os.path.join(ROOT, 'data_batch_%d' % (b, ))
    #print(f)
    X, Y = load_CIFAR_batch(f)
    xs.append(X)    
    ys.append(Y)    
  Xtr = np.concatenate(xs)
  Ytr = np.concatenate(ys)
  del X, Y
  Xte, Yte = load_CIFAR_batch(os.path.join(ROOT, 'test_batch'))
  return Xtr, Ytr, Xte, Yte

def get_CIFAR10_data(num_training=5000, num_validation=500, num_test=500):
    """
    Load the CIFAR-10 dataset from disk and perform preprocessing to prepare
    it for classifiers. These are the same steps as we used for the SVM, but
    condensed to a single function.
    """
    # Load the raw CIFAR-10 data    
    cifar10_dir = 'D://cifar-10-python//cifar-10-batches-py//'
    X_train, y_train, X_test, y_test = load_CIFAR10(cifar10_dir)
    #print(X_train.shape)
    # Subsample the data
    mask = range(num_training, num_training + num_validation)
    print(mask)
    print("X_train")
    print(X_train.shape)
    X_val = X_train[mask]
    print("X_val")
    print(X_val.shape)
    y_val = y_train[mask]
    mask = range(num_training)
    print(mask)
    print("X_train:num_training1")
    print(X_train.shape)
    X_train = X_train[mask]
    print("X_train:num_training2")
    print(X_train.shape)
    y_train = y_train[mask]
    mask = range(num_test)
    X_test = X_test[mask]
    y_test = y_test[mask]

    # Normalize the data: subtract the mean image
    mean_image = np.mean(X_train, axis=0)
    X_train -= mean_image
    X_val -= mean_image
    X_test -= mean_image
    
    # Transpose so that channels come first
    X_train = X_train.transpose(0, 3, 1, 2).copy()
    X_val = X_val.transpose(0, 3, 1, 2).copy()
    X_test = X_test.transpose(0, 3, 1, 2).copy()

    # Package data into a dictionary
    return {
      'X_train': X_train, 'y_train': y_train,
      'X_val': X_val, 'y_val': y_val,
      'X_test': X_test, 'y_test': y_test,
    }
    

其中，

def load_CIFAR10(ROOT):
  """ load all of cifar """
  xs = []
  ys = []
  for b in range(1,2):
    f = os.path.join(ROOT, 'data_batch_%d' % (b, ))
    X, Y = load_CIFAR_batch(f)
    xs.append(X)
    ys.append(Y)    
  Xtr = np.concatenate(xs)
  Ytr = np.concatenate(ys)
  del X, Y
  Xte, Yte = load_CIFAR_batch(os.path.join(ROOT, 'test_batch'))
  return Xtr, Ytr, Xte, Yte

load_CIFAR10()函数中，append()函数是向列表list中添加一个元素或多个元素，numpy库中concatenate()函数的作用是，沿着一个坐标轴连接多个数组。

    mask = range(num_training, num_training + num_validation)
    print(mask)
    print("X_train")
    print(X_train.shape)
    X_val = X_train[mask]
    print("X_val")
    print(X_val.shape)
    y_val = y_train[mask]
    mask = range(num_training)
    print(mask)
    print("X_train:num_training1")
    print(X_train.shape)
    X_train = X_train[mask]
    print("X_train:num_training2")
    print(X_train.shape)
    y_train = y_train[mask]
    mask = range(num_test)
    X_test = X_test[mask]
    y_test = y_test[mask]

主要完成数据切分的工作，打印结果为：

range(5000, 5500)
X_train
(10000, 32, 32, 3)
X_val
(500, 32, 32, 3)
range(0, 5000)
X_train:num_training1
(10000, 32, 32, 3)
X_train:num_training2
(5000, 32, 32, 3)

因为训练模型时，输入的数据格式为（batch_size,channel,height,width)从X_trian的shape中我们看到，（10000，32，32，3）,10000是数据条数，32分别是高度和宽度，3是channel即信号通道，所以在训练模型之前需要进行如下操作：

    # Transpose so that channels come first
    X_train = X_train.transpose(0, 3, 1, 2).copy()
    X_val = X_val.transpose(0, 3, 1, 2).copy()
    X_test = X_test.transpose(0, 3, 1, 2).copy()