tensorflow(神经网络)学习笔记(三)之调参数实战（笔记）

tensorboard

# 指定目录
LOG_DIR = '.'
run_label = 'run_vgg_tensorboard_fine_tune'
run_dir = os.path.join(LOG_DIR, run_label)
if not os.path.join(run_dir):
    os.mkdir(run_dir)
train_log_dir = os.path.join(run_dir, 'train')
test_log_dir = os.path.join(run_dir, 'test')


# 命名name
conv1_1 = tf.layers.conv2d(x_image,
                           32, # output channel number
                           (3,3), # kernel size
                           padding = 'same',
                           activation = tf.nn.relu,
                           name = 'conv1_1')
                    

def variable_summary(var, name):
    """Constructs summary for statistics of a variable"""
    with tf.name_scope(name):
        mean = tf.reduce_mean(var)
        with tf.name_scope('stddev'):
            stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean)))
        tf.summary.scalar('mean', mean)
        tf.summary.scalar('stddev', stddev)
        tf.summary.scalar('min', tf.reduce_min(var))
        tf.summary.scalar('max', tf.reduce_max(var))
        tf.summary.histogram('histogram', var)

with tf.name_scope('summary'):
    variable_summary(conv1_1, 'conv1_1')
    variable_summary(conv1_2, 'conv1_2')
    variable_summary(conv2_1, 'conv2_1')
    variable_summary(conv2_2, 'conv2_2')
    variable_summary(conv3_1, 'conv3_1')
    variable_summary(conv3_2, 'conv3_2')
...
# 写入
    train_writer = tf.summary.FileWriter(train_log_dir, sess.graph)
    test_writer = tf.summary.FileWriter(test_log_dir)

     train_summary_str = eval_ops_results[-1]
     train_writer.add_summary(train_summary_str, i+1)
     test_summarys_str = sess.run([megred_summary_test],
                                     feed_dict={
                                         x: fixed_test_batch_data,
                                         y: fixed_test_batch_labels,    
                                     })[0]
     test_writer.add_summary(test_summarys_str, i+1)
    
   

3.tensorboard --host=*** --logdir=文件目录

fine-tune(微改其他人的模型)

1.save models

model_dir = os.path.join(run_dir, 'model')
if not os.path.exists(model_dir):
    os.mkdir(model_dir)
saver = tf.train.Saver()
...
if (i+1) % output_model_every_steps == 0:
      saver.save(sess, os.path.join(model_dir, 'ckp-{}'.format(i+1)))
      print('model saved to ckp-{}'.format(i+1))

2.restore models

model_name = 'ckp-1000'
model_path = os.path.join(model_dir, model_name)
....
    if os.path.exists(model_path + '.index'):
        saver.restore(sess, model_path)
        print('model restore from {}'.format(model_path))
    else:
        print('model {} does not exist'.format(model_path))
    

• activation:relu, sigmoid,tanh
• weight initializer: he,xavier,normal
• optimzier:Adam,Momentum,Gradient Descent

---

def convent(inputs, activation, kernel_initializer):
        
    conv1_1 = tf.layers.conv2d(inputs,
                               32, # output channel number
                               (3,3), # kernel size
                               padding = 'same',
                               activation = tf.nn.relu,
                               kernel_initializer = kernel_initializer
                               name = 'conv1_1'
                               )
# activation :tf.nn.relu,tf.nn.sigmoid,tf.nn.tanh
# kernel_initializer:默认为tf.glorot_normal_initializer,
# tf.truncated_normal_initializer(stddev=0.02)
sigmoid:53.39% vs relu:73.35% on 10k train
tf.glorot_normal_initializer 76.53%, 100k train.
tf.truncated_normal_initializer(stddev=0.02) :78.04% 100k train
tf.keras.initializers.he_normal: 71.52%			100k train (deep net)

with tf.name_scope('train_op'):
    train_op = tf.train.AdamOptimizer(1e-3).minimize(loss)    # 78.04%
    tf.train.GradientDescentOptimizer(1e-4).minimize(loss)	  # 12.35%
    # reason: 1. initializer incorrect. 2.不充分的训练
    tf.train.MomentumOptimizer(learning_rate=1e-4, momentum=0.9).minimize(loss) #35.75%
    

数据增强api

• resize (缩放图像)
• crop (裁剪)
• flip (翻转)
• brightness (改变光照) & contrast (改变对比度)

基本库

import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.pyplot import  imshow

resize api

tf.image.resize_area 
tf.image.resize_bicubic # 图像缩放,双线性算法插入
tf.image.resize_nearest_neighbor # 图像放大,使用相近的像素点插入放大的像素点

# 图片放大
img_string = tf.read_file(picture1)
# img_string2 = tf.read_file(picture2)
img_decoded = tf.image.decode_image(img_string)
# sess多图片,使用四维矩阵,[index, 宽, 长, 通道数]
# resize

img_decoded = tf.reshape(img_decoded, [1, 1200, 1400, 3])
# 缩小
resize_img = tf.image.resize_bicubic(img_decoded, [2400, 2800])
sess = tf.Session()
img_decoded_val = sess.run(resize_img)
img_decoded_val = img_decoded_val.reshape((2400, 2800, 3))
img_decoded_val = np.asarray(img_decoded_val, np.uint8)
print(img_decoded_val.shape)

imshow(img_decoded_val)
# imshow(img_decoded_val2)

pylab.show()

# crop 裁剪
tf.image.pad_to_bounding_box()
tf.image.crop_to_bounding_box()
tf.random_crop()

img_decoded = tf.image.decode_image(img_string)

img_decoded = tf.reshape(img_decoded, [1, 1200, 1400, 3])
# crop
padded_img = tf.image.pad_to_bounding_box(img_decoded, 100, 200, 1500, 1600) # arg:[0:2]画布中的位置,[2:4]画布大小
sess = tf.Session()
img_decoded_val = sess.run(padded_img)
img_decoded_val = img_decoded_val.reshape((1500, 1600, 3))
img_decoded_val = np.asarray(img_decoded_val, np.uint8)
print(img_decoded_val.shape)

imshow(img_decoded_val)
# imshow(img_decoded_val2)

pylab.show()

flip

tf.image.flip_up_down()
tf.image.flip_left_right()
tf.image.random_flip_left_right()
tf.image.random_flip_up_down()

ex:
padded_img = tf.image.random_flip_left_right(img_decoded) 

brightness

tf.image.adjust_brightness()
tf.image.random_brightness()
tf.image.adjust_contrast()
tf.image.random_contrast()

ex:
padded_img = tf.image.adjust_brightness(img_decoded, -0.5) # -0.5 为光强减少50%

实战

# train 100k: 78.04%  -> 82.6%
x_image = tf.reshape(x, [-1, 3, 32, 32])
x_image = tf.transpose(x_image, perm=[0, 2, 3, 1])
data_aug_1 = tf.image.random_flip_left_right(x_image)
data_aug_2 = tf.image.random_brightness(data_aug_1, max_delta=63)
data_aug_3 = tf.image.random_contrast(data_aug_2, lower=0.2, upper=1.8)

normal_data_aug_3 = data_aug_3 / 127.5 - 1

继续加深网络层次

conv1_1 = tf.layers.conv2d(normal_data_aug_3,
                           32, # output channel number
                           (3,3), # kernel size
                           padding = 'same',
                           activation = tf.nn.relu,
                           name = 'conv1_1')
conv1_2 = tf.layers.conv2d(conv1_1,
                           32, # output channel number
                           (3,3), # kernel size
                           padding = 'same',
                           activation = tf.nn.relu,
                           name = 'conv1_2')
conv1_3 = tf.layers.conv2d(conv1_2,
                           32, # output channel number
                           (3,3), # kernel size
                           padding = 'same',
                           activation = tf.nn.relu,
                           name = 'conv1_3')

# 16 * 16
pooling1 = tf.layers.max_pooling2d(conv1_3,
                                   (2, 2), # kernel size
                                   (2, 2), # stride
                                   name = 'pool1')


conv2_1 = tf.layers.conv2d(pooling1,
                           32, # output channel number
                           (3,3), # kernel size
                           padding = 'same',
                           activation = tf.nn.relu,
                           name = 'conv2_1')
conv2_2 = tf.layers.conv2d(conv2_1,
                           32, # output channel number
                           (3,3), # kernel size
                           padding = 'same',
                           activation = tf.nn.relu,
                           name = 'conv2_2')
conv2_3 = tf.layers.conv2d(conv2_2,
                           32, # output channel number
                           (3,3), # kernel size
                           padding = 'same',
                           activation = tf.nn.relu,
                           name = 'conv2_3')
# 8 * 8
pooling2 = tf.layers.max_pooling2d(conv2_3,
                                   (2, 2), # kernel size
                                   (2, 2), # stride
                                   name = 'pool2')

conv3_1 = tf.layers.conv2d(pooling2,
                           32, # output channel number
                           (3,3), # kernel size
                           padding = 'same',
                           activation = tf.nn.relu,
                           name = 'conv3_1')
conv3_2 = tf.layers.conv2d(conv3_1,
                           32, # output channel number
                           (3,3), # kernel size
                           padding = 'same',
                           activation = tf.nn.relu,
                           name = 'conv3_2')
conv3_3 = tf.layers.conv2d(conv3_2,
                           32, # output channel number
                           (3,3), # kernel size
                           padding = 'same',
                           activation = tf.nn.relu,
                           name = 'conv3_3')
# 4 * 4 * 32
pooling3 = tf.layers.max_pooling2d(conv3_2,
                                   (2, 2), # kernel size
                                   (2, 2), # stride
                                   name = 'pool3')

归一化

train 100K :85.6%

def conv_wrapper(inputs, 
                 name, 
                 is_training,
                 output_channel=32, 
                 kernel_size=(3,3),
                 activation=tf.nn.relu,
                 padding = 'same'):
    """wrapper of tf.laysers.conv2d"""
    # without bn: conv  -> activation
    # with batch normalization: conv -> bn -> activation
    with tf.name_scope(name):
        conv2d = tf.layers.conv2d(inputs,
                            output_channel,
                            kernel_size,
                            padding= padding,
                            activation= None,
                            name= name+'/conv2d')
        bn = tf.layers.batch_normalization(conv2d,
                                           training= is_training)
        return activation(bn)

def pooling_warpper(inputs, name):
    return tf.layers.max_pooling2d(inputs,
                                   (2,2),
                                   (2,2),
                                   name=name)
# conv1: 神经元图， feature_map, 输出图像

conv1_1 = conv_wrapper(normal_data_aug_3, name = 'conv1_1', is_training=True)
conv1_2 = conv_wrapper(conv1_1, name = 'conv1_2', is_training=True)
conv1_3 = conv_wrapper(conv1_2, name = 'conv1_3', is_training=True)
pooling1 = pooling_warpper(conv1_3, name = 'pool1')


conv2_1 = conv_wrapper(pooling1, name = 'conv2_1', is_training=True)
conv2_2 = conv_wrapper(conv2_1, name = 'conv2_2', is_training=True)
conv2_3 = conv_wrapper(conv2_2, name = 'conv2_3', is_training=True)
pooling2 = pooling_warpper(conv2_3, name = 'pool2')


conv3_1 = conv_wrapper(normal_data_aug_3, name = 'conv3_1', is_training=True)
conv3_2 = conv_wrapper(conv3_1, name = 'conv3_2', is_training=True)
conv3_3 = conv_wrapper(conv3_2, name = 'conv3_3', is_training=True)
pooling3 = pooling_warpper(conv3_3, name = 'pool1')

使用resnet train acc 67% after fine-tune -> 94%