theano-xnor-net代码注释9 pylearn2/cifar10.py

"""
.. todo::

    WRITEME
"""
import os
import logging

import numpy
from theano.compat.six.moves import xrange

from pylearn2.datasets import cache, dense_design_matrix
from pylearn2.expr.preprocessing import global_contrast_normalize
from pylearn2.utils import contains_nan
from pylearn2.utils import serial
from pylearn2.utils import string_utils


_logger = logging.getLogger(__name__)


class CIFAR10(dense_design_matrix.DenseDesignMatrix):

    """
    .. todo::

        WRITEME

    Parameters
    ----------
    which_set : str
        One of 'train', 'test'
    center : WRITEME
    rescale : WRITEME
    gcn : float, optional
        Multiplicative constant to use for global contrast normalization.
        No global contrast normalization is applied, if None
    start : WRITEME
    stop : WRITEME
    axes : WRITEME
    toronto_prepro : WRITEME
    preprocessor : WRITEME
    """

    def __init__(self, which_set, center=False, rescale=False, gcn=None,
                 start=None, stop=None, axes=('b', 0, 1, 'c'),
                 toronto_prepro = False, preprocessor = None):
        # note: there is no such thing as the cifar10 validation set;
        # pylearn1 defined one but really it should be user-configurable
        # (as it is here)

        self.axes = axes

        # we define here:
        dtype = 'uint8'
        ntrain = 50000
        nvalid = 0  # artefact, we won't use it
        ntest = 10000

        # we also expose the following details:
        self.img_shape = (3, 32, 32)
        #self.img_size存的是图片元素个数，3*32*32=3072
        self.img_size = numpy.prod(self.img_shape)
        #类别为10,0-9对应标签为label_names
        self.n_classes = 10
        self.label_names = ['airplane', 'automobile', 'bird', 'cat', 'deer',
                            'dog', 'frog', 'horse', 'ship', 'truck']

        # prepare loading
        #fnames为一个列表，存的是data_batch1～5
        fnames = ['data_batch_%i' % i for i in range(1, 6)]
        #datasets为一个空字典
        datasets = {}
        #${PYLEARN2_DATA_PATH}已经在配置pylearn之后存在个人系统目录.bashrc中了，为/home/ubuntu/pylearn2-data/data
        # datapath里存/home/ubuntu/pylearn2-data/data/cifar10/cifar-10-batches-py/
        datapath = os.path.join(
            string_utils.preprocess('${PYLEARN2_DATA_PATH}'),
            'cifar10', 'cifar-10-batches-py')
        #在data_batch1～5+test_batch六个文件中做循环
        for name in fnames + ['test_batch']:
            #当前fname为当前操作数据集文件的全路径
            fname = os.path.join(datapath, name)
            #如果文件不存在，raise一个error
            if not os.path.exists(fname):
                raise IOError(fname + " was not found. You probably need to "
                              "download the CIFAR-10 dataset by using the "
                              "download script in "
                              "pylearn2/scripts/datasets/download_cifar10.sh "
                              "or manually from "
                              "http://www.cs.utoronto.ca/~kriz/cifar.html")
            #将当前数据集文件快速缓存进datasets字典中
            datasets[name] = cache.datasetCache.cache_file(fname)
        #lenx数值就是50000
        lenx = int(numpy.ceil((ntrain + nvalid) / 10000.) * 10000)
        #设置一个全0矩阵x大小50000×3072,y大小50000×1的np.array
        x = numpy.zeros((lenx, self.img_size), dtype=dtype)
        y = numpy.zeros((lenx, 1), dtype=dtype)

        # load train data
        #下载训练集
        nloaded = 0
        #enumerate返回的是（引索值，当前迭代对象）
        for i, fname in enumerate(fnames):
            #将括号内信息存入log文件
            _logger.info('loading file %s' % datasets[fname])
            #从刚加载好的datasets字典中取出当前操作文件数据，存入data，python版本的cifar10本身就是一个字典，
            # 所以当前data就是一个字典，字典中有batch_label,labels,data,filenames四种信息
            data = serial.load(datasets[fname])
            #一个数据集中有10000个图片信息，对应data为10000个3072的np.array，labels对应10000个一维标签，依次取出5个对应训练数据集文件，按照顺序依次存入x与y
            x[i * 10000:(i + 1) * 10000, :] = data['data']
            y[i * 10000:(i + 1) * 10000, 0] = data['labels']
            #以下三行代码运行不到，在迭代完5个文件时候nloaded=50000,小于60000,此时循环就已经退出
            nloaded += 10000
            if nloaded >= ntrain + nvalid + ntest:
                break

        # load test data
        #加载测试集合
        #将括号内信息存入log文件
        _logger.info('loading file %s' % datasets['test_batch'])
        #加载'test_batch'测试集数据，存入data，前面data中信息已经清空
        data = serial.load(datasets['test_batch'])
        #重组数据
        # process this data
        #Xs为一个字典，‘train’关键字中存训练集的50000条图像数据，‘test’关键字中存测试集的10000条图像数据
        #Ys为一个字典，‘train’关键字中存训练集的50000个标签，‘test’关键字中存测试集的10000个标签
        Xs = {'train': x[0:ntrain],
              'test': data['data'][0:ntest]}

        Ys = {'train': y[0:ntrain],
              'test': data['labels'][0:ntest]}
        #which_set为调用CIFAR10类时候传如的参数，选择是[train、test]
        #即X为对应[train or test]的图像数据
        #y为对应[train or test]的标签

        X = numpy.cast['float32'](Xs[which_set])
        y = Ys[which_set]
        #在该数据集中标签的存储为一个列表list，该行代码是要将label转化为与data一样的ndarray格式
        if isinstance(y, list):
            y = numpy.asarray(y).astype(dtype)
        #如果测试数据集标签数不为10000,重新整理为（y.shape[0], 1）大小
        if which_set == 'test':
            assert y.shape[0] == 10000
            y = y.reshape((y.shape[0], 1))

        if center:
            X -= 127.5
        self.center = center

        if rescale:
            X /= 127.5
        self.rescale = rescale

        if toronto_prepro:
            assert not center
            assert not gcn
            X = X / 255.
            if which_set == 'test':
                other = CIFAR10(which_set='train')
                oX = other.X
                oX /= 255.
                X = X - oX.mean(axis=0)
            else:
                X = X - X.mean(axis=0)
        self.toronto_prepro = toronto_prepro

        self.gcn = gcn
        if gcn is not None:
            gcn = float(gcn)
            X = global_contrast_normalize(X, scale=gcn)

        if start is not None:
            # This needs to come after the prepro so that it doesn't
            # change the pixel means computed above for toronto_prepro
            assert start >= 0
            assert stop > start
            assert stop <= X.shape[0]
            X = X[start:stop, :]
            y = y[start:stop, :]
            assert X.shape[0] == y.shape[0]

        if which_set == 'test':
            assert X.shape[0] == 10000

        view_converter = dense_design_matrix.DefaultViewConverter((32, 32, 3),
                                                                  axes)

        super(CIFAR10, self).__init__(X=X, y=y, view_converter=view_converter,
                                      y_labels=self.n_classes)

        assert not contains_nan(self.X)

        if preprocessor:
            preprocessor.apply(self)

    def adjust_for_viewer(self, X):
        """
        .. todo::

            WRITEME
        """
        # assumes no preprocessing. need to make preprocessors mark the
        # new ranges
        rval = X.copy()

        # patch old pkl files
        if not hasattr(self, 'center'):
            self.center = False
        if not hasattr(self, 'rescale'):
            self.rescale = False
        if not hasattr(self, 'gcn'):
            self.gcn = False

        if self.gcn is not None:
            rval = X.copy()
            for i in xrange(rval.shape[0]):
                rval[i, :] /= numpy.abs(rval[i, :]).max()
            return rval

        if not self.center:
            rval -= 127.5

        if not self.rescale:
            rval /= 127.5

        rval = numpy.clip(rval, -1., 1.)

        return rval

    def __setstate__(self, state):
        super(CIFAR10, self).__setstate__(state)
        # Patch old pkls
        if self.y is not None and self.y.ndim == 1:
            self.y = self.y.reshape((self.y.shape[0], 1))
        if 'y_labels' not in state:
            self.y_labels = 10

    def adjust_to_be_viewed_with(self, X, orig, per_example=False):
        """
        .. todo::

            WRITEME
        """
        # if the scale is set based on the data, display X oring the
        # scale determined by orig
        # assumes no preprocessing. need to make preprocessors mark
        # the new ranges
        rval = X.copy()

        # patch old pkl files
        if not hasattr(self, 'center'):
            self.center = False
        if not hasattr(self, 'rescale'):
            self.rescale = False
        if not hasattr(self, 'gcn'):
            self.gcn = False

        if self.gcn is not None:
            rval = X.copy()
            if per_example:
                for i in xrange(rval.shape[0]):
                    rval[i, :] /= numpy.abs(orig[i, :]).max()
            else:
                rval /= numpy.abs(orig).max()
            rval = numpy.clip(rval, -1., 1.)
            return rval

        if not self.center:
            rval -= 127.5

        if not self.rescale:
            rval /= 127.5

        rval = numpy.clip(rval, -1., 1.)

        return rval

    def get_test_set(self):
        """
        .. todo::

            WRITEME
        """
        return CIFAR10(which_set='test', center=self.center,
                       rescale=self.rescale, gcn=self.gcn,
                       toronto_prepro=self.toronto_prepro,
                       axes=self.axes)