CIFAR10代码解释

Copyright 2015 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the “License”);
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an “AS IS” BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

在这之前，都是一些许可证之类的东西

----------

建立cifar10识别网络
"""Builds the CIFAR-10 network.
功能摘要
Summary of available functions:

计算用于训练的输入图像和标签。 如果您想运行评估，请改用input()。
 # Compute input images and labels for training. If you would like to run
 # evaluations, use inputs() instead.
 方法：inputs, labels = distorted_inputs()

计算模型输入的推断以进行预测。
 # Compute inference on the model inputs to make a prediction.
 方法：predictions = inference(inputs)

计算与标签相关的预测总损失。
 # Compute the total loss of the prediction with respect to the labels.
 方法：loss = loss(predictions, labels)

创建一个Graph，对损失进行一次训练。
 # Create a graph to run one step of training with respect to the loss.
 方法：train_op = train(loss, global_step)
"""


----------


# pylint: disable=missing-docstring
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import re
import sys
import tarfile

from six.moves import urllib
import tensorflow as tf

import cifar10_input

## 前面是导入相关库 ##


----------

设置一个全局变量存储器FLAGS，方便从命令行读取参数
FLAGS = tf.app.flags.FLAGS

基本模型参数
# Basic model parameters.
第一个参数是参数名称，第二个参数是默认值，第三个参数是参数描述，可以用FLAGS.参数名称来调用
设置batch-size大小，默认128
tf.app.flags.DEFINE_integer('batch_size', 128, """Number of images to process in a batch.""")
设置数据默认存储路径
tf.app.flags.DEFINE_string('data_dir', '/tmp/cifar10_data', """Path to the CIFAR-10 data directory.""")
设置fp16半精度浮点运算默认为False
tf.app.flags.DEFINE_boolean('use_fp16', False, """Train the model using fp16.""")

描述CIFAR-10数据集的全局常量。
# Global constants describing the CIFAR-10 data set.
图片尺寸
IMAGE_SIZE = cifar10_input.IMAGE_SIZE
分类数量
NUM_CLASSES = cifar10_input.NUM_CLASSES
每一代训练样本数（50000）
NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN = cifar10_input.NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN
每一代测试样本数（10000）
NUM_EXAMPLES_PER_EPOCH_FOR_EVAL = cifar10_input.NUM_EXAMPLES_PER_EPOCH_FOR_EVAL

描述训练过程的常量
# Constants describing the training process.
移动平均值衰减（用于移动平均线的衰减）
MOVING_AVERAGE_DECAY = 0.9999     # The decay to use for the moving average.
学习率下降之后的每一代的数量
NUM_EPOCHS_PER_DECAY = 350.0      # Epochs after which learning rate decays.
学习率衰减因子
LEARNING_RATE_DECAY_FACTOR = 0.1  # Learning rate decay factor.
初始学习率
INITIAL_LEARNING_RATE = 0.1       # Initial learning rate.

如果使用多个GPU训练模型，请在所有Op名称前加上tower_name，以区分操作。 请注意，在可视化模型时，会从摘要的名称中删除此前缀。
# If a model is trained with multiple GPUs, prefix all Op names with tower_name
# to differentiate the operations. Note that this prefix is removed from the
# names of the summaries when visualizing a model.
TOWER_NAME = 'tower'

cifar10数据下载地址
DATA_URL = 'http://www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz'


def _activation_summary(x):
     帮助使用者去创建活动摘要
  """Helper to create summaries for activations.

  创建提供激活直方图的摘要
  Creates a summary that provides a histogram of activations.
  创建一个衡量激活稀疏性的摘要
  Creates a summary that measures the sparsity of activations.

  输入x是一个张量，没有返回值
  Args:
    x: Tensor
  Returns:
    nothing
  """
  如果这是一个多GPU训练，请从名称中删除'tower_ [0-9] /'。 这有助于在tensorboard上清晰呈现。
  # Remove 'tower_[0-9]/' from the name in case this is a multi-GPU training
  # session. This helps the clarity of presentation on tensorboard.
  re.sub(pattern，repl，string)是利用正则表达式对输入的字符串进行替换，并返回处理后的字符串
  这里是将'x.op.name'里面的TOWER_NAME[0-9]用''替换
  tensor_name = re.sub('%s_[0-9]*/' % TOWER_NAME, '', x.op.name)
  tf.summary.histogram(name, value)将一个张量在训练中的分布情况以直方图的形式在TensorBoard直方图仪表板上显示．name：生成的节点名称.作为TensorBoard中的一个系列名称.values：一个实数张量.用于构建直方图的值.
  tf.summary.histogram(tensor_name + '/activations', x)
  tf.summary.scalar(name,tensor,collections=None,family=None)对标量数据汇总和记录
  tf.summary.scalar(tensor_name + '/sparsity', tf.nn.zero_fraction(x))


def _variable_on_cpu(name, shape, initializer):
     帮助创建存储在CPU内存中的变量。
  """Helper to create a Variable stored on CPU memory.

  传入变量名、形状、初始值，返回一个张量
  Args:
    name: name of the variable
    shape: list of ints
    initializer: initializer for Variable

  Returns:
    Variable Tensor
  """
  tf.device(device_name)指定运行设备，其中'0'指设备号，此时Tensor存储在内存里，不在显存里
  with tf.device('/cpu:0'):
    如果默认设置里面设置的计算精度为fp16，则数据类型为tf.float16，否则dtype设置为tf.float32
    dtype = tf.float16 if FLAGS.use_fp16 else tf.float32
    tf.get_variable(name,  shape, initializer, dtype)，获取已存在的变量（要求不仅名字，而且初始化方法等各个参数都一样），如果不存在，就新建一个，name就是变量的名称，shape是变量的维度，initializer是变量初始化的方式, dtype是数据类型
    var = tf.get_variable(name, shape, initializer=initializer, dtype=dtype)
  return var


def _variable_with_weight_decay(name, shape, stddev, wd):
     帮助创建具有权重衰减的初始化变量
  """Helper to create an initialized Variable with weight decay.
  请注意，变量初始化为截断的正态分布。仅在指定了一个权重衰减时才添加权重衰减。
  截断正态分布简单来说就是x有限制的正太分布
  Note that the Variable is initialized with a truncated normal distribution.
  A weight decay is added only if one is specified.

  函数传输入变量名、形状、高斯分布的标准差、是否添加L2损失权重衰减乘以此浮点数，返回一个张量
  Args:
    name: name of the variable
    shape: list of ints
    stddev: standard deviation of a truncated Gaussian
    wd: add L2Loss weight decay multiplied by this float. If None, weight
        decay is not added for this Variable.

  Returns:
    Variable Tensor
  """
  设置数据类型
  dtype = tf.float16 if FLAGS.use_fp16 else tf.float32
  获取创建在cup中的变量
  var = _variable_on_cpu(
      name,
      shape,
      tf.truncated_normal_initializer(stddev=stddev, dtype=dtype))
  if wd is not None:
    权重衰减
    weight_decay = tf.multiply(tf.nn.l2_loss(var), wd, name='weight_loss')
    tf.add_to_collection：把变量放入一个集合，把很多变量变成一个列表
    tf.add_to_collection('losses', weight_decay)
  return var


def distorted_inputs():
     使用Reader ops为CIFAR训练构建扭曲的输入
  """Construct distorted input for CIFAR training using the Reader ops.

  返回值为images,labels
  Returns:
    images: Images. 4D tensor of [batch_size, IMAGE_SIZE, IMAGE_SIZE, 3] size.
    labels: Labels. 1D tensor of [batch_size] size.

  如果数据文件路径错误，抛出异常
  Raises:
    ValueError: If no data_dir
  """
  if not FLAGS.data_dir:
    raise ValueError('Please supply a data_dir')
  从默认设置中提取出数据文件的路径
  data_dir = os.path.join(FLAGS.data_dir, 'cifar-10-batches-bin')
  从文件中提取出iamges和labels
  images, labels = cifar10_input.distorted_inputs(data_dir=data_dir,batch_size=FLAGS.batch_size)

  将images,labels编码转换为系统编码float16
  if FLAGS.use_fp16:
    images = tf.cast(images, tf.float16)
    labels = tf.cast(labels, tf.float16)
  return images, labels


def inputs(eval_data):
     使用Reader ops为CIFAR测试构造输入
  """Construct input for CIFAR evaluation using the Reader ops.

    输入为布尔值，指示是否应该使用训练或测试数据集
  Args:
    eval_data: bool, indicating if one should use the train or eval data set.

  Returns:
    images: Images. 4D tensor of [batch_size, IMAGE_SIZE, IMAGE_SIZE, 3] size.
    labels: Labels. 1D tensor of [batch_size] size.

  Raises:
    ValueError: If no data_dir
  """
  if not FLAGS.data_dir:
    raise ValueError('Please supply a data_dir')
  data_dir = os.path.join(FLAGS.data_dir, 'cifar-10-batches-bin')
  images, labels = cifar10_input.inputs(eval_data=eval_data,
                                        data_dir=data_dir,
                                        batch_size=FLAGS.batch_size)
  if FLAGS.use_fp16:
    images = tf.cast(images, tf.float16)
    labels = tf.cast(labels, tf.float16)
  return images, labels


def inference(images):
     建立cifar10识别模型
  """Build the CIFAR-10 model.

  输入从distorted_inputs()或inputs()返回的iamges， 返回Logits
  Args:
    images: Images returned from distorted_inputs() or inputs().

  Returns:
    Logits.
  """
   我们用tf.get_variable()将所有变量实例化而不是用tf.Variable()，为了在多个GPU训练运行中共享变量
   如果只在一个GPU上跑这个模型，我们可以通过用tf.Variable()替换所有的tf.get_variable()来简化这个方法
  # We instantiate all variables using tf.get_variable() instead of
  # tf.Variable() in order to share variables across multiple GPU training runs.
  # If we only ran this model on a single GPU, we could simplify this function
  # by replacing all instances of tf.get_variable() with tf.Variable().
  #
  # conv1
  ## 第一个卷积层 ##
  tf.variable('conv1')创建一个叫conv1的命名空间
  with tf.variable_scope('conv1') as scope:
    卷积核
    kernel = _variable_with_weight_decay('weights',
                                         shape=[5, 5, 3, 64],
                                         stddev=5e-2,
                                         wd=0.0)
    使用tf.nn.conv2d()对对象进行卷积                          
    conv = tf.nn.conv2d(images, kernel, [1, 1, 1, 1], padding='SAME')
    偏置
    biases = _variable_on_cpu('biases', [64], tf.constant_initializer(0.0))
    将偏置加到conv上
    pre_activation = tf.nn.bias_add(conv, biases)
    使用relu进行激活
    conv1 = tf.nn.relu(pre_activation, name=scope.name)
    创建第一次卷积的摘要
    _activation_summary(conv1)

  # pool1
  最大池化
  pool1 = tf.nn.max_pool(conv1, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1],
                         padding='SAME', name='pool1')
  # norm1
  lrn(全称：local response normalization--局部响应标准化)作为relu激励之后防止数据过拟合而提出的一种处理方法
  norm1 = tf.nn.lrn(pool1, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75,
                    name='norm1')

  # conv2
  第二层卷积
  with tf.variable_scope('conv2') as scope:
    kernel = _variable_with_weight_decay('weights',
                                         shape=[5, 5, 64, 64],
                                         stddev=5e-2,
                                         wd=0.0)
    conv = tf.nn.conv2d(norm1, kernel, [1, 1, 1, 1], padding='SAME')
    biases = _variable_on_cpu('biases', [64], tf.constant_initializer(0.1))
    pre_activation = tf.nn.bias_add(conv, biases)
    conv2 = tf.nn.relu(pre_activation, name=scope.name)
    _activation_summary(conv2)

  # norm2
  norm2 = tf.nn.lrn(conv2, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75,
                    name='norm2')
  # pool2
  pool2 = tf.nn.max_pool(norm2, ksize=[1, 3, 3, 1],
                         strides=[1, 2, 2, 1], padding='SAME', name='pool2')

  # local3
  全连接层
  with tf.variable_scope('local3') as scope:
    将所有内容排成一列，以便我们可以执行单个矩阵乘法。
    # Move everything into depth so we can perform a single matrix multiply.
    矩阵变形
    reshape = tf.reshape(pool2, [FLAGS.batch_size, -1])
    获取列数
    dim = reshape.get_shape()[1].value
    weights = _variable_with_weight_decay('weights', shape=[dim, 384],
                                          stddev=0.04, wd=0.004)
    biases = _variable_on_cpu('biases', [384], tf.constant_initializer(0.1))
    local3 = relu(W*x + b)
    local3 = tf.nn.relu(tf.matmul(reshape, weights) + biases, name=scope.name)
    _activation_summary(local3)

  # local4
  全连接层
  with tf.variable_scope('local4') as scope:
    weights = _variable_with_weight_decay('weights', shape=[384, 192],
                                          stddev=0.04, wd=0.004)
    biases = _variable_on_cpu('biases', [192], tf.constant_initializer(0.1))
    local4 = tf.nn.relu(tf.matmul(local3, weights) + biases, name=scope.name)
    _activation_summary(local4)

  我们不在这里应用softmax，因为tf.nn.sparse_softmax_cross_entropy_with_logits接受未缩放的logits并在内部执行softmax以提高效率。
  # linear layer(WX + b),
  # We don't apply softmax here because
  # tf.nn.sparse_softmax_cross_entropy_with_logits accepts the unscaled logits
  # and performs the softmax internally for efficiency.
  with tf.variable_scope('softmax_linear') as scope:
    weights = _variable_with_weight_decay('weights', [192, NUM_CLASSES],
                                          stddev=1/192.0, wd=0.0)
    biases = _variable_on_cpu('biases', [NUM_CLASSES],
                              tf.constant_initializer(0.0))
    softmax_linear = tf.add(tf.matmul(local4, weights), biases, name=scope.name)
    _activation_summary(softmax_linear)

  return softmax_linear


def loss(logits, labels):
    将L2损失添加到所有可训练变量中
  """Add L2Loss to all the trainable variables.

  添加“损失”和“损失/平均”的摘要
  Add summary for "Loss" and "Loss/avg".
  输入从inference()返回的logits和从distorted_inputs()或inputs()返回的labels，返回float类型的损失
  Args:
    logits: Logits from inference().
    labels: Labels from distorted_inputs or inputs(). 1-D tensor
            of shape [batch_size]

  Returns:
    Loss tensor of type float.
  """
    计算批次中的平均交叉熵损失
  # Calculate the average cross entropy loss across the batch.
  将labels转换为int64
  labels = tf.cast(labels, tf.int64)
  计算交叉熵
  cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
      labels=labels, logits=logits, name='cross_entropy_per_example')
  求均值，即平均交叉熵
  cross_entropy_mean = tf.reduce_mean(cross_entropy, name='cross_entropy')
  交叉熵损失列表
  tf.add_to_collection('losses', cross_entropy_mean)

  总损失定义为交叉熵损失加上所有权重衰减项（L2损失）。
  # The total loss is defined as the cross entropy loss plus all of the weight
  # decay terms (L2 loss).
  tf.add_n能将多个列表对应位置相加，tf.get_collection()返回列表名称为losses的列表
  return tf.add_n(tf.get_collection('losses'), name='total_loss')


def _add_loss_summaries(total_loss):
     添加CIFAR-10模型中的损失摘要
  """Add summaries for losses in CIFAR-10 model.

  生成所有损失的移动平均值和相关摘要，使网络的表现可视化
  Generates moving average for all losses and associated summaries for
  visualizing the performance of the network.

  传入从loss()中返回的损失，返回用于生成损失的移动平均线。
  Args:
    total_loss: Total loss from loss().
  Returns:
    loss_averages_op: op for generating moving averages of losses.
  """
    计算所有单个损失的移动平均值和总损失。
  # Compute the moving average of all individual losses and the total loss.
   指数加权平均
  loss_averages = tf.train.ExponentialMovingAverage(0.9, name='avg')
  losses = tf.get_collection('losses')
  loss_averages_op = loss_averages.apply(losses + [total_loss])

  附上所有单个损失和总损失的标量摘要; 为损失的平均版本做同样的事情。
  # Attach a scalar summary to all individual losses and the total loss; do the
  # same for the averaged version of the losses.
  for l in losses + [total_loss]:
     将每个损失命名为“(raw)”，并将损失的移动平均版本命名为原始损失名称。
    # Name each loss as '(raw)' and name the moving average version of the loss
    # as the original loss name.
    标量数据汇总和记录
    tf.summary.scalar(l.op.name + ' (raw)', l)
    tf.summary.scalar(l.op.name, loss_averages.average(l))

  return loss_averages_op


def train(total_loss, global_step):
     训练cifar10模型
  """Train CIFAR-10 model.

  创建优化程序并应用于所有可训练变量。 为所有可训练变量添加移动平均值。
  Create an optimizer and apply to all trainable variables. Add moving
  average for all trainable variables.

  传入损失及训练步骤数
  Args:
    total_loss: Total loss from loss().
    global_step: Integer Variable counting the number of training steps
      processed.
  Returns:
    train_op: op for training.
  """
  影响学习率的变量
  # Variables that affect learning rate.
  每一代的batch数：总训练数/单个batch大小
  num_batches_per_epoch = NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN / FLAGS.batch_size
  衰减速度
  decay_steps = int(num_batches_per_epoch * NUM_EPOCHS_PER_DECAY)

   根据步数以指数方式衰减学习率
  # Decay the learning rate exponentially based on the number of steps.
   函数实现指数衰减学习率
  lr = tf.train.exponential_decay(INITIAL_LEARNING_RATE,
                                  global_step,
                                  decay_steps,
                                  LEARNING_RATE_DECAY_FACTOR,
                                  staircase=True)
  tf.summary.scalar('learning_rate', lr)

   生成所有损失和相关摘要的移动平均值
  # Generate moving averages of all losses and associated summaries.
  loss_averages_op = _add_loss_summaries(total_loss)

   计算梯度
  # Compute gradients.
  tf.control_dependencies()指定某些操作执行的依赖关系,返回一个上下文管理器，只有将括号里的的内容执行了，才会执行里面的内容
  with tf.control_dependencies([loss_averages_op]):
    梯度下降
    opt = tf.train.GradientDescentOptimizer(lr)
    grads = opt.compute_gradients(total_loss)

    应用梯度
  # Apply gradients.
  apply_gradient_op = opt.apply_gradients(grads, global_step=global_step)

    添加可训练变量的直方图
  # Add histograms for trainable variables.
    tf.trainable_variables返回的是需要训练的变量列表
  for var in tf.trainable_variables():
    tf.summary.histogram(var.op.name, var)

    添加梯度的直方图
  # Add histograms for gradients.
  for grad, var in grads:
    if grad is not None:
      tf.summary.histogram(var.op.name + '/gradients', grad)

    跟踪所有可训练变量的移动平均值
  # Track the moving averages of all trainable variables.
  variable_averages = tf.train.ExponentialMovingAverage(
      MOVING_AVERAGE_DECAY, global_step)
  variables_averages_op = variable_averages.apply(tf.trainable_variables())

  with tf.control_dependencies([apply_gradient_op, variables_averages_op]):
    train_op = tf.no_op(name='train')

  return train_op


def maybe_download_and_extract():
     从Alex的网站下载并解压tarball
  """Download and extract the tarball from Alex's website."""
  dest_directory = FLAGS.data_dir
  if not os.path.exists(dest_directory):
    os.makedirs(dest_directory)
  filename = DATA_URL.split('/')[-1]
  filepath = os.path.join(dest_directory, filename)
  if not os.path.exists(filepath):
    def _progress(count, block_size, total_size):
      sys.stdout.write('\r>> Downloading %s %.1f%%' % (filename,
          float(count * block_size) / float(total_size) * 100.0))
      刷新输出
      sys.stdout.flush()
    filepath, _ = urllib.request.urlretrieve(DATA_URL, filepath, _progress)
    print()
    statinfo = os.stat(filepath)
    print('Successfully downloaded', filename, statinfo.st_size, 'bytes.')
  extracted_dir_path = os.path.join(dest_directory, 'cifar-10-batches-bin')
  if not os.path.exists(extracted_dir_path):
    tarfile.open(filepath, 'r:gz').extractall(dest_directory)