TensorFlow 2.0 笔记（七）—— 卷积神经网络

Feature maps

2D Convolution
$$y(t)=x(t)\ast h(t)={\int }_{-\infty }^{\infty }x(\tau )h(t-\tau )d\tau$$

Padding & Stride

import tensorflow as tf

x = tf.random.normal([1, 32, 32, 3])
layer = tf.keras.layers.Conv2D(4, kernel_size=5, strides=1, padding='valid')
out = layer(x)

layer = tf.keras.layers.Conv2D(4, kernel_size=5, strides=1, padding='same')
out = layer(x)

layer = tf.keras.layers.Conv2D(4, kernel_size=5, strides=2, padding='same')
out = layer(x)

#tf.nn.conv2d
x = tf.random.normal([1, 32, 32, 3])
w = tf.random.normal([5, 5, 3, 4])
b = tf.zeros([4])

out = tf.nn.conv2d(x, w, strides=1, padding='VALID')
out = out + b
out = tf.nn.conv2d(x, w, strides=2, padding='VALID')

Pooling

• Max pooling
• Avg pooling

x = tf.random.normal([1, 14, 14, 4])
pool = tf.keras.layers.MaxPool2D(2, strides=2)
out = pool(x)

pool = tf.keras.layers.MaxPool2D(3, strides=2)
out = pool(x)

out = tf.nn.max_pool2d(x, 2, strides=2, padding='VALID')

UpSampling2D

x = tf.random.normal([1, 7, 7, 4])
layer = tf.keras.layers.UpSampling2D(size=3)
out = layer(x)

layer = tf.keras.layers.UpSampling2D(size=2)
out = layer(x)

Relu

#relu
x = tf.random.normal([2, 3])
tf.nn.relu(x)

tf.keras.layers.ReLU()(x)

VGG13

import os
import tensorflow as tf
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
#set gpu memory, otherwise => UnknownError: Failed to get convolution algorithm. This is probably because
physical_devices = tf.config.experimental.list_physical_devices('GPU')
assert len(physical_devices) > 0 #"Not enough GPU hardware devices available"
tf.config.experimental.set_memory_growth(physical_devices[0], True)

tf.random.set_seed(2345)

conv_layers = [ # 5 units of conv + max pooling
    # unit 1
    tf.keras.layers.Conv2D(64, kernel_size=[3, 3], padding="same", activation=tf.nn.relu),
    tf.keras.layers.Conv2D(64, kernel_size=[3, 3], padding="same", activation=tf.nn.relu),
    tf.keras.layers.MaxPool2D(pool_size=[2, 2], strides=2, padding="same"),

    # unit 2
    tf.keras.layers.Conv2D(128, kernel_size=[3, 3], padding="same", activation=tf.nn.relu),
    tf.keras.layers.Conv2D(128, kernel_size=[3, 3], padding="same", activation=tf.nn.relu),
    tf.keras.layers.MaxPool2D(pool_size=[2, 2], strides=2, padding="same"),

    # unit 3
    tf.keras.layers.Conv2D(256, kernel_size=[3, 3], padding="same", activation=tf.nn.relu),
    tf.keras.layers.Conv2D(256, kernel_size=[3, 3], padding="same", activation=tf.nn.relu),
    tf.keras.layers.MaxPool2D(pool_size=[2, 2], strides=2, padding="same"),

    # unit 4
    tf.keras.layers.Conv2D(512, kernel_size=[3, 3], padding="same", activation=tf.nn.relu),
    tf.keras.layers.Conv2D(512, kernel_size=[3, 3], padding="same", activation=tf.nn.relu),
    tf.keras.layers.MaxPool2D(pool_size=[2, 2], strides=2, padding="same"),

    # unit 5
    tf.keras.layers.Conv2D(512, kernel_size=[3, 3], padding="same", activation=tf.nn.relu),
    tf.keras.layers.Conv2D(512, kernel_size=[3, 3], padding="same", activation=tf.nn.relu),
    tf.keras.layers.MaxPool2D(pool_size=[2, 2], strides=2, padding="same")
]


def preprocess(x, y):
    x = tf.cast(x, dtype=tf.float32) / 255.
    y = tf.cast(y, dtype=tf.int32)
    return x,y


(x, y), (x_test, y_test) = tf.keras.datasets.cifar100.load_data()
y = tf.squeeze(y, axis=1)
y_test = tf.squeeze(y_test, axis=1)
print(x.shape, y.shape, x_test.shape, y_test.shape)

train_data = tf.data.Dataset.from_tensor_slices((x, y))
train_data = train_data.shuffle(1000).map(preprocess).batch(64)

test_data = tf.data.Dataset.from_tensor_slices((x_test, y_test))
test_data = test_data.map(preprocess).batch(64)

sample = next(iter(train_data))
print('sample:', sample[0].shape, sample[1].shape,
      tf.reduce_min(sample[0]), tf.reduce_max(sample[0]))


def main():
    # [b, 32, 32, 3] -> [b, 1, 1, 512]
    conv_model = tf.keras.Sequential(conv_layers)
    fc_model = tf.keras.Sequential([
        tf.keras.layers.Dense(256, activation=tf.nn.relu),
        tf.keras.layers.Dense(128, activation=tf.nn.relu),
        tf.keras.layers.Dense(100)
    ])

    conv_model.build(input_shape=[None, 32, 32, 3])
    fc_model.build(input_shape=[None, 512])
    optimizer = tf.keras.optimizers.Adam(lr=1e-4)

    # [1, 2] + [3, 4] => [1, 2, 3, 4]
    variables = conv_model.trainable_variables + fc_model.trainable_variables
    acc_m = tf.keras.metrics.Accuracy()
    loss_m = tf.keras.metrics.Mean()

    for epoch in range(50):
        for step, (image, label) in enumerate(train_data):
            with tf.GradientTape() as tape:
                # [b, 32, 32, 3] => [b, 1, 1, 512]
                out = conv_model(image)
                # flatten, => [b, 512]
                out = tf.reshape(out, [-1, 512])
                # [b, 512] => [b, 100]
                logits = fc_model(out)
                # [b] => [b, 100]
                y_onehot = tf.one_hot(label, depth=100)
                # compute loss
                loss = tf.losses.categorical_crossentropy(y_onehot, logits, from_logits=True)
                loss = tf.reduce_mean(loss)
                loss_m.update_state(loss)

            grads = tape.gradient(loss, variables)
            optimizer.apply_gradients(zip(grads, variables))

            if step % 100 == 0:
                for x_t, y_t in test_data:
                    out = conv_model(x_t)
                    out = tf.reshape(out, [-1, 512])
                    logits = fc_model(out)
                    prob = tf.nn.softmax(logits, axis=1)
                    pred = tf.argmax(prob, axis=1)
                    acc_m.update_state(y_t, pred)
                print(epoch, step, 'loss:', loss_m.result().numpy(),
                      'evaluate Acc:', acc_m.result().numpy())
                loss_m.reset_states()
                acc_m.reset_states()


if __name__ == '__main__':
    main()

UnknownError: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above. [Op:Conv2D]

import tensorflow as tf
physical_devices = tf.config.experimental.list_physical_devices('GPU')
assert len(physical_devices) > 0 #"Not enough GPU hardware devices available"
tf.config.experimental.set_memory_growth(physical_devices[0], True)

经典卷积神经网络

Batch Norm
Feature scaling

• Image Normalization

img_mean = tf.constant([0.485, 0.456, 0.406])
img_std = tf.constant([0.229, 0.224, 0.225])
def normalize(x, mean=img_mean, std=img_std):
    """

    :param x: [b, h, w, c]
    :param mean: [b, h, w, c]
    :param std: [c]
    :return:
    """
    x = (x - mean) / std
    return x

• Batch Normalization

  • dynamic mean/std

  • Batch Norm
  • Layer Norm
  • Instance Norm
  • Group Norm

  • layers.BatchNormalization
    • net = tf.keras.layers.BatchNormalization()
      • axis=-1
      • center=True
      • scale=True
      • trainable=True
    • net(x, training=None)

net = tf.keras.layers.BatchNormalization()
x = tf.random.normal([2, 3])
out = net(x)

net.trainable_variables
net.variables