def train():
with tf.Graph().as_default():
global_step=tf.Variable(0,trainable=False)
# 输入选用的是distored_inputs函数
images,labels=cifar10.distorted_inputs()
logits=cifar10.inference(images)
loss=cifar10.loss(logits,labels)
train_op=cifar10.train(loss,global_step)
# Create a saver
saver=tf.train.Saver(tf.all_variables())
summary_op=tf.merge_all_summaries()
init=tf.initialize_all_variables()
# Start running operations on the Graph
sess=tf.Session(config=tf.ConfigProto(
log_device_placement=FLAGS.log_device_placement))
sess.run(init)
tf.train.start_queue_runners(sess=sess)#Start the queue runners
summary_write=tf.train.SummaryWrite(FLAGS,train_dir,sess.gragh)
#在最高的迭代步骤数里进行循环迭代
for step in xrange(FLAGS.max_steps):
start_time=time.time()
_,loss_value=sess.run([trian_op,loss])
duration=time.time()-start_time
assert not np.isnan(loss_value),#'Model diverged with loss = NaN'
#每10个输入数据显示次step,loss,时间等运行数据
if step % 10 ==0:
num_exmples_per_step=FLAGS.batch_size
examples_per_sec=num_exmples_per_step /duration
sec_per_batch=float(duration)
format_str=('%s:step %d, loss=%.2f (%.1f example/sec; %.3f)'
'sec/batch')
print (format_str % (datatime.now(),step,loss_value,
examples_per_sec,sec_per_batch))
#每100个输入数据将网络的状况体现在summary里
if step %100==0:
summary_str=sess.run(summary_op)
summary_writer.add_summary(summary_str,step)
#Save the model checkpoint periodically
#每1000个输入数据保存次模型
if step%1000== 0 or (step+1)==FLAGS.max_steps:
checkpoint_path=os.path.join(FALGS.train_dir,'model.ckpt')
saver.save(sess,checkpoint_path,global_step=step)
def distorted_inputs(data_dir,batch_size):
"""Construct distorted input for CIFAR training using the Reader ops.
Args:
data_dir:Path to the CIFAR-10 data directory.
batch_size:Number of images per batch.
return:
images:Image.4D tensor of [batch_size,Image_SIZE,3] size
labels:Labels.1D tensor of [batch_size] size
"""
filenames=[os.path.join(data_dir,'data_batch_%d.bin' %i)
for i in xrange(1,6)]
for f in filenames:
if not tf.gfile.Exists(f):
raise ValueError('Failed to find file: '+f)
#Create a queue that produces the filenames to read
filename_queue=tf.train.string_input_producer(filenames)
#Read examples from files in the filename queue.
read_input=read_cifar10(filename_queue)
reshaped_image=tf.cast(read_input.uint8image, tf.float32)
height=Image_SIZE
width=Image_SIZE
#Rondomly crop a [height,width] section of the image
distorted_image=tf.random_crop(reshaped_image,[height,width,3])
#Rondomly flip the image horizontally
#步骤2:随机颠倒图片的左右,概率为50%
distorted_image=tf.image.randomly_flip_left_right(distorted_image)
#步骤3:随机改变图片的亮度以及色彩对比
distorted_image=tf.image.random_brightness(distorted_image,max_delta=63)
distorted_image=tf.image.random_contrast(distorted_image,lower=0.2,upper=1.8)
#Subtract off the mean and divide by the variance of the pixels.
float_image=tf.image.per_imaeg_whitening(distorted_image)
#Ensure that the random shuffling has good mixing properties
min_fraction_of_examples_in_queue=0.4
min_queue_examples=int(NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN*
min_fraction_of_examples_in_queue)
print('Filling queue with %d CIFAR images befor starting to train.'
'This will take a few minutes.' %min_queue_examples)
#Generate a batch of images and labels by building up a queue of examples.
return _generate_image_and_label_batch(float_image, read_input.label,
min_queue_examples, batch_size,
shuffle=True)
"""
最后一段的意思为在queue里有了不少于40%的数据的时候训练才能开始。那么在测试的时候,我们需要经过这个步骤么?答案是非也。
在cifar10_input.py文件里,distorded_inputs函数的下方,一个名为inputs的函数代表了输入被运用在eval时的逻辑。
在输入参数方面,这个inputs函数在保留了distorded_inputs的同时增加了一个名为eval_data的参数,
一个bool参数代表了是运用训练的数据还是测试的数据。下面,让我们来大概看下这个函数的逻辑
"""
def inputs(eval_data, data_dir, batch_size):
"""Construct input for CIFAR evaluation using the Reader ops.
Args:
eval_data: bool,indicating if one should use the train or eval data set.
data_dir: path to the CIFAR-10 data directory.
batch_size: Number of images per batch.
Returns:
images:Image.4D tensor of [batch_size,IMAGE_SIZE, IMAGE_SIZE,3] size.
labels:Labels.1D tensor of [batch_size] size
"""
if not eval_data:
filenames=[os.path.join(data_dir, 'data_batch_%d.bin'%i)
for i in xrange(1,6)]
num_examples_per_epoch= NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN
else:
filenames=[os.path.join(data_dir, 'test_batch.bin')]
num_examples_per_epoch= NUM_ EXAMPLES_PER_EPOCH_FOR_EVAL
for f in filenames:
if not tf.gfile.Exists(f):
raise ValueError('Failed to find file: '+f)
#Create a queue that produces the filenames to read.
filename_queue = tf.train.string_input_producer(filenames)
#Read examples from files in the filename queue.
read_input = read_cifar10(filename_queue)
reshaped_image = tf.cast(read_input.uint8image, tf.float32)
height = IMAGE_SIZE
width= IMAGE_SIZE
#Image processing for evaluation
#Crop the central [height,width] of the image.<br> #截取图片中心区域
resized_image= tf.image.resize_image_with_crop_or_pad( reshaped_image,
width,height)
#Subtract off the mean and divide by the variance of the pixels.<br>#平衡图片的色差
float_image=tf.image.per_image_whitening(resized_image)
#Ensure that the random shuffling has good mixing properties.
min_fraction_of_examples_in_queue=0.4
min_queue_examples= int(num_examples_per_epoch*
min_fraction_of_examples_in_queue )
#Generate a batch of images and labels by building up a queue of examples.
return _generate_image_and_label_batch(float_image,read_input.label,
min_queue_examples,batch_size,
shuffle=False)
"""
讲了关于输入的小窍门,我们应该来看看具体的CNN模型了。
如何制造一个CNN模型呢?
让我们先来看一个简单的版本,即MNIST教程里的模型:
"""
#The varibles below hold all the trainable weights. They are passes an initial value which will be assigned when we call;
# {tf.initialize_all_variables().run()}
conv1_weights=tf.variable(
tf.truncated_normal([5,5,NUM_CHANNELS,32], #5*5 filter,depth 32.
stddev=0.1,
seed=SEED, dtype=data_type()))
conv1_biases=tf.Variable(tf.zeros([32],dtype=data_type()))
conv2_weights=tf.Variable(
tf.truncated_normal([5,5,32,64],
stddev=0.1,
seed=SEED,dtype=data_type()))
conv2_biases=tf.Variable(tf.constant(0.1,shape=[64],dtype=data_type()))
fc1_weights= tf.Variable( # full connected, depth 512.
tf.truncated_normal([IMAGE_SIZE // 4 * IMAGE_SIZE // 4*64,512],
stddev=0.1,
seed=SEED, dtype=data_type()))
fc1_biases= tf.Variable(tf.constant(0.1,shape=[512],dtype=data_type()))
fc2_weights= tf.Variable(tf.truncated_normal([512,NUM_LABELS],
stddev=0.1,
seed=SEED,dtype=data_type()))
fc2_biases=tf.Variable(tf.constant(
0.1,shape=[NUM_LABELS],dtype=data_type()))
"""上面是参数变量,下面是网络结构"""
# We will replicate the model structure for the training subgraph, as well
# as the evaluation subgraphs, while sharing the trainable parameters.
def model(data,train=False):
"""The model definition."""
#2D convolution, with 'SAME' padding(i.e. the output feature map has the same size as the input)
#Note that {strides} is a 4D array whose shape matches the data layout:[image index,y,x,depth].
conv =tf.nn.conv2d(data,
conv1_weights,
strides=[1,1,1,1],
padding='SAME'
)
#Bias and rectified linear non-linearity.
relu = tf.nn.relu(tf.nn.bias_add(conv,conv1_biases))
pool = tf.nn.max_pool(relu,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME')
conv = tf.nn.conv2d(pool,
conv2_weights,
strides=[1, 1, 1, 1],
padding='SAME')
relu = tf.nn.relu(tf.nn.bias_add(conv, conv2_biases))
pool = tf.nn.max_pool(relu,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME')
# Reshape the feature map cuboid into a 2D matrix to feed it to the
# fully connected layers.
pool_shape=pool.get_shape().as_list()
reshape=tf.reshape(pool,[pool_shape[0],pool_shape[1]*pool_shape[2]*pool_shape[3]])
#Fully connected layer.
#Note that the '+' operation automatically broadcasts the biases.
hidden=tf.nn.relu(tf.matmul(reshape,fc1_weights)+fc1_biases)
#Add a 50% dropout during training only.
#Dropout also scales activations suvh that no rescaling is needed at evaluation time.
if train:
hidden = tf.nn.dropout(hidden,0.5,seed=SEED)
return tf.matmul(hidden,fc2_weights)+fc2_biases
#Training computation: logits +cross-entropy loss.
logits = model(train_data_node,True)
loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits, train_labels_node))
#L2 regularization for the fully connected parameters.
regularizers = (tf.nn.l2_loss(fc1_weights)+tf.nn.l2_loss(fc1_biases)+
tf.nn.l2_loss(fc2_weights)+tf.nn.12_loss(fc2_biases))
#Add the regularization term to the loss.
loss += 5e-4 *regularizers
#Optimizer: set up a variable that's incremented once per batch and controls the learning rate decay.
batch = tf.Variable(0,dtype=data_type))
#Decay once per epoch, using an exponential schedule starting at 0.01
learning_rate=tf.train.exponential_decay(
0.01, #Base learning rate.
batch*BATCH_SIZE, #Current index into the dataset.
train_size, #Decay step.
0.95, #Decay rate.
staircase=TRUE )
#Use simple momentum for the optimization
optimizer = tf.train.momentumOptimizer(learning_rate,0.9).minimize(loss,global_step=batch)
#Predictions for the current training minibatch
train_prediction = tf.nn.softmax(logits) #TensorFlow provides several operations that help you perform classification.
#Predictions for the test and validation, which we'll compute less often.
eval_prediction = tf.nn.softmax(model(eval_data))
"""
这段代码很直白,在定义了convolution1,convolution2,fully_connected1和fully_connected2
层神经网络的weight和biases参数后,
在模型函数里,我们通过conv2d, relu, max_pool等方式在两次重复后将得到的结果重新整理后
输入那个fully connected的神经网络中,即matmul(reshape,fc1_weights) + fc1_biases。
之后再经历了第二层的fully connected net后得到logits。
定义loss以及optimizer等常见的过程后结果是由softmax来取得。
这个逻辑我们在CIFAR10里也会见到,它的表达如下:
"""
def inference(images):
"""Build the cifar_10 model
Args:#实参
images:Images returned from distorted_inputs() or inputs
Returns:
Logits.#模型的逻辑输出
"""
# we instantiate all variables using tf.get_variable() instead of tf.Variable()
# in order to share variables across multiple GPU trainig runs.
# If we only ran this model on a single GPU,we could simplify this function by
# replacing all instances of tf.get_variable() with tf.Variable()
#conv1
with tf.variable_scope('conv1') as scope:
kernel=_variable_with_weight_decay('weights',shape=[5,5,3,64],
stddev=1e-4,wd=0.0)
conv=tf.nn.conv2d(images,kernel,[1,1,1,1],padding='SAME')
#输入的图片由于是彩图,有三个channel,所以在conv2d中,我们规定输出为64个channel的feature map。
biases=_variable_on_cpu('biases',[64], tf.constant_initializer(0.0))
bias=tf.nn.bias_add(conv,biases)
conv1=tf.nn.relu(bias,name=scope.name)
_activation_summary(conv1)
#pool1
pool1=tf.nn.max_pool(conv1,ksize=[1,3,3,1],strides=[1,2,2,1],
padding='SAME',name='pool1')
#norm1
norm1=tf.nn.lrn(pool1,4,bias=1.0,alpha=0.001/9.0,beta=0.75,name='norm1')
#conv2
with tf.variable_scope('conv2') as scope:
#由于之前的输出是64个channel,即我们这里的输入,输出我们也规定为64
kernel=_variable_with_weight_decay('weights',shape=[5,5,64,64],
stddev=1e-4, wd=0.0)
conv = tf.nn.conv2d(norm1,kernel,[1,1,1,1],padding='SAME')
biases= _variable_on_cpu('biases',[64],tf.constant_initializer(0.1))
bias= tf.nn.bias_add(conv,biases)
conv2= tf.nn.relu(bias,name=scope.name)
_activation_summary(conv2)
#norm2
norm2= tf.nn.lrn(conv2,4,bias=1.0,alpha=0.001/9.0,beta=0.75,name='norm2')
#pool2
pool2= tf.nn.maxpool(norm2,ksize=[1,3,3,1],
strides=[1,2,2,1],padding='SAME',name='pool2')
#local3
with tf.variable_scope('local3') as scope:
#Moving everthing into depth so we can perform a single matrix multiply.
reshape = tf. rashape(pool2,[FLAGS.batch_size,-1])
dim =reshape.get_shape()[1].value
# 这里之前在reshape时的那个-1是根据tensor的大小自动定义为batch_size和剩下的,所以我们剩下的就是一张图的所有内容,
# 我们将它训练并map到384个神经元节点上
weights=_variable_with_weight_decay('weights',shape=[dim,384],
stddev=0.04,wd=0.004)
biases =_variable_on_cpu('biases',[384],tf.constant_initializer(0.1))
local3= tf.nn.relu(tf.matmul(reshape,weights)+biases,name=scope.name)
_activation_summary(local3)
#local4
with tf.variable_scope('local4') as scope:
#由于我们之前的节点有384个,这里我们进一步缩减为192个。
weights= _variable_with_weight_decay('weights',shape=[384,192],
stddev=0.04,wd=0.004)
baises= _variable_on_cpu('biases',[192],tf.constant_initializer(0.1))
local4= tf.nn.relu(tf.matmul(local3,weights)+biases,name=scope.name)
_activation_summary(local4)
#softmax,i.e. softmax(WX+b)
with tf.variable_scope('softmax_linear') as scope:
# 这是softmax输出时的网络,我们由192个节点map到输出的不同数量上,这里假设
# 有10类,我们就输出10个num_classes。
weights= _variable_with_weight_decay('weights',[192,NUM_CLASS],
stddev=1/192.0,wd=0.0)
biases= _variable_on_cpu('biases',[NUM_CLASS],
tf.constant_initializer(0.0) )
softmax_linear=tf.add(tf.matmul(local4,weights),biases,name=scope.name)
_activation_summary(softmax_linear)
return softmax_linear
"""
这里的逻辑跟之前的在框架上基本一样,不同在哪里呢?首先,这次我们的输入是彩图。
学过图片处理的朋友肯定知道彩图有3个channel,而之前MNIST只是单个channel的灰白图。
所以,在我们制作feature map的时候,由1个channel map到了32个(注,那个NUM_CHANNELS是1)。这里我们不过把NUM_CHANNELS给直接写为了3而已。
另外,我们还运用了variable scope,这是一种很好的方式来界定何时对那些变量进行分享,同时,我们也不需要反复定义weight和biases的名字了。
"""
""""对Loss的定义由loss函数写明,其内容无非是运用了sparse_softmax_corss_entropy_with_logits,
基本流程同于MNIST,这里将不详细描述。最后,cifar10.py里的train函数虽然逻辑很简单,但是也有值得注意的地方。代码如下:
"""
def train(total_loss,global_step):
"""Train CIFAR-10 model.
Creat an optimizer and apply to all trainable variables.
add moving average for all trainable variables.
Args:
total_loss:Total loss from loss()
global_step: Integer Variable counting the number of training steps processed.
Returns:
train_op: op for training.
"""
#Variables that affect learning rate.
num_batches_per_epoch = NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN/ FLAGS.batch_size
decay_steps= int(num_batches_epoch_per_epoch*NUM_EPOCHS_PER_DECAY)
#Decay the learning rate exponentially based on the number of steps.
lr= tf.train.exponential_decay(INITIAL_LEARNING_RATE,
global_step,
decay_steps,
LEARNING_RATE_DECAY_FACTOR,
staircase=True)
tf.scalar_summary('learning_rate',lr)
#Generate moving averages of all losses and associated summaries.
loss_averages_op =_add_loss_summaries(total_loss)
#Compute gradients
# control dependencies 的运用,这里只有loss_averages_op完成了,我们才会进行gradient descent的优化
with tf.control_dependencies([loss_averages_op]):
opt= tf.train.GradientDescentOptimizer(lr)
grads= opt.compute_gradients(total_loss)
#Apply gradients
apply_gradient_op =opt.apply_gradients(grads,global_step=global_step)
#add histograms for trainable variables
for var in tf.trainable_variables():
tf.histogram_summary(var.op.name,var)
#add histograms for gradients.
for grad,var in grads:
if grad is not None:
tf.histogram_summary(var.op.name+'/gradients',grad)
#track the moving averages of all trainable variables.
variable_averages= tf.train.ExponentialMovingAverage(
MOVING_AVERAGE_DECAY,global_step )
variables_average_op= variable_averages.apply(tf.trainable_variables)
with tf.control_dependencies([apply_gradient_op,variables_average_op]):
train_op= tf.no_op(name='train')
return train_op
cifar-10
最新推荐文章于 2023-01-15 20:15:51 发布