Tensorflow复现AlexNet网络

复现代码

代码已跑通

import tensorflow as tf
from tensorflow.keras import layers
import math
import time
from datetime import datetime
batch_size = 32
num_batches = 100

w_init = 0
b_init = 0

class Conv1(layers.Layer):
    def __init__(self):
        super(Conv1, self).__init__()

    def build(self, input_shape):
        w_init = tf.random_normal_initializer()
        b_init = tf.zeros_initializer()
        self.kernel = self.add_weight(name='Conv1/kernel', shape=[11, 11, 3, 96], initializer=w_init, dtype='float32', trainable=True)
        self.biases = self.add_weight(name='Conv1/biases', shape=[96], initializer=b_init, dtype='float32', trainable=True)

    def call(self, inputs):
        conv = tf.nn.conv2d(inputs, self.kernel, [1, 4, 4, 1], padding='SAME')
        relu = tf.nn.relu(tf.nn.bias_add(conv, self.biases))
        lrn = tf.nn.lrn(relu, 4, bias=1.0, alpha=0.001/9.0, beta=0.75, name='Conv1/lrn')
        pool = tf.nn.max_pool(lrn, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='VALID', name='Conv1/pool')

        return pool


class Conv2(layers.Layer):
    def __init__(self):
        super(Conv2, self).__init__()

    def build(self, input_shape):
        w_init = tf.random_normal_initializer()
        b_init = tf.zeros_initializer()
        self.kernel = self.add_weight(name='Conv2/kernel', shape=[5, 5, 96, 256], initializer=w_init, dtype='float32',
                                      trainable=True)
        self.biases = self.add_weight(name='Conv2/biases', shape=[256], initializer=b_init, dtype='float32',
                                      trainable=True)

    def call(self, inputs):
        conv = tf.nn.conv2d(inputs, self.kernel, [1, 1, 1, 1], padding='SAME')
        relu = tf.nn.relu(tf.nn.bias_add(conv, self.biases))
        lrn = tf.nn.lrn(relu, 4, bias=1.0, alpha=0.001/9.0, beta=0.75, name='Conv2/lrn')
        pool = tf.nn.max_pool(lrn, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='VALID', name='Conv2/pool')

        return pool

class Conv3(layers.Layer):
    def __init__(self):
        super(Conv3, self).__init__()

    def build(self, input_shape):
        w_init = tf.random_normal_initializer()
        b_init = tf.zeros_initializer()
        self.kernel = self.add_weight(name='Conv3/kernel', shape=[3, 3, 256, 384], initializer=w_init, dtype='float32',
                                      trainable=True)
        self.biases = self.add_weight(name='Conv3/biases', shape=[384], initializer=b_init, dtype='float32',
                                      trainable=True)

    def call(self, inputs):
        conv = tf.nn.conv2d(inputs, self.kernel, [1, 1, 1, 1], padding='SAME')
        relu = tf.nn.relu(tf.nn.bias_add(conv, self.biases))

        return relu

class Conv4(layers.Layer):
    def __init__(self):
        super(Conv4, self).__init__()

    def build(self, input_shape):
        w_init = tf.random_normal_initializer()
        b_init = tf.zeros_initializer()
        self.kernel = self.add_weight(name='Conv4/kernel', shape=[3, 3, 384, 384], initializer=w_init, dtype='float32',
                                      trainable=True)
        self.biases = self.add_weight(name='Conv4/biases', shape=[384], initializer=b_init, dtype='float32',
                                      trainable=True)

    def call(self, inputs):
        conv = tf.nn.conv2d(inputs, self.kernel, [1, 1, 1, 1], padding='SAME')
        relu = tf.nn.relu(tf.nn.bias_add(conv, self.biases))

        return relu

class Conv5(layers.Layer):
    def __init__(self):
        super(Conv5, self).__init__()

    def build(self, input_shape):
        w_init = tf.random_normal_initializer()
        b_init = tf.zeros_initializer
        self.kernel = self.add_weight(name='Conv5/kernel', shape=[3, 3, 384, 256], initializer=w_init, dtype='float32',
                                      trainable=True)
        self.biases = self.add_weight(name='Conv5/biases', shape=[256], initializer=b_init, dtype='float32',
                                      trainable=True)

    def call(self, inputs):
        conv = tf.nn.conv2d(inputs, self.kernel, [1, 1, 1, 1], padding='SAME')
        relu = tf.nn.relu(tf.nn.bias_add(conv, self.biases))
        pool = tf.nn.max_pool(relu, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='VALID', name='Conv5/pool')

        return pool

class AlexNet(tf.keras.Model):
    def __init__(self):
        super(AlexNet, self).__init__()
        self.conv1 = Conv1()
        self.conv2 = Conv2()
        self.conv3 = Conv3()
        self.conv4 = Conv4()
        self.conv5 = Conv5()
        self.flatten = layers.Flatten()
        self.dense1 = layers.Dense(units=4096, activation='relu')
        self.dense2 = layers.Dense(units=4096, activation='relu')

    def call(self, inputs):
        x = self.conv1(inputs)
        x = self.conv2(x)
        x = self.conv3(x)
        x = self.conv4(x)
        x = self.conv5(x)
        x = self.flatten(x)
        x = self.dense1(x)
        x = self.dense2(x)

        return x

image_size = 224
image_shape = [batch_size, image_size, image_size, 3]
image_init = tf.random_normal_initializer(stddev=1e-1)
image_data = tf.Variable(initial_value=image_init(shape=image_shape), dtype='float32')

alexnet = AlexNet()
alexnet(image_data)
alexnet.summary()

# 评测模型前向传播的耗时
total_dura = 0.0
total_dura_squared = 0.0
for step in range(num_batches + 10):
    start_time = time.time()
    alexnet(image_data)
    duration = time.time() - start_time
    if step > 10:
        if step%10==0:
            print('%s: step %d, duration=%.3f' % (datetime.now(), step-10, duration))
        total_dura += duration
        total_dura_squared += duration*duration
average_time = total_dura/num_batches

print('%s:Forward across %d steps, %.3f +/- %.3f sec/batch' % (datetime.now(), num_batches, average_time,\
                                                               math.sqrt(total_dura_squared/num_batches-average_time*average_time)))

# 反向传播耗时
back_total_dura = 0.0
back_total_dura_squared = 0.0

for step in range(num_batches + 10):
    start_time = time.time()
    with tf.GradientTape() as tape:
        loss = tf.nn.l2_loss(alexnet(image_data))
        gradients = tape.gradient(loss, alexnet.trainable_variables)
        duration = time.time() - start_time
    if step > 10:
        if step % 10 == 0:
            print('%s: step %d, duration=%.3f' % (datetime.now(), step - 10, duration))
        back_total_dura += duration
        back_total_dura_squared += duration * duration
back_avg_t = back_total_dura/num_batches
print('%s:Forward-backward across %d steps, %.3f +/- %.3f sec/batch' % (datetime.now(), num_batches, back_avg_t,\
                                                               math.sqrt(back_total_dura_squared/num_batches-back_avg_t*back_avg_t)))

原开源代码

import tensorflow as tf
import math
import time
from datetime import datetime

batch_size = 32
num_batches = 100


# 在函数inference_op()内定义前向传播的过程
def inference_op(images):
    parameters = []

    # 在命名空间conv1下实现第一个卷积层
    with tf.name_scope("conv1"):
        kernel = tf.Variable(tf.truncated_normal([11, 11, 3, 96], dtype=tf.float32,
                                                 stddev=1e-1), name="weights")
        conv = tf.nn.conv2d(images, kernel, [1, 4, 4, 1], padding="SAME")
        biases = tf.Variable(tf.constant(0.0, shape=[96], dtype=tf.float32),
                             trainable=True, name="biases")
        conv1 = tf.nn.relu(tf.nn.bias_add(conv, biases))

        # 打印第一个卷积层的网络结构
        print(conv1.op.name, ' ', conv1.get_shape().as_list())

        parameters += [kernel, biases]

    # 添加一个LRN层和最大池化层
    lrn1 = tf.nn.lrn(conv1, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75, name="lrn1")
    pool1 = tf.nn.max_pool(lrn1, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1],
                           padding="VALID", name="pool1")

    # 打印池化层网络结构
    print(pool1.op.name, ' ', pool1.get_shape().as_list())

    # 在命名空间conv2下实现第二个卷积层
    with tf.name_scope("conv2"):
        kernel = tf.Variable(tf.truncated_normal([5, 5, 96, 256], dtype=tf.float32,
                                                 stddev=1e-1), name="weights")
        conv = tf.nn.conv2d(pool1, kernel, [1, 1, 1, 1], padding="SAME")
        biases = tf.Variable(tf.constant(0.0, shape=[256], dtype=tf.float32),
                             trainable=True, name="biases")
        conv2 = tf.nn.relu(tf.nn.bias_add(conv, biases))
        parameters += [kernel, biases]

        # 打印第二个卷积层的网络结构
        print(conv2.op.name, ' ', conv2.get_shape().as_list())

    # 添加一个LRN层和最大池化层
    lrn2 = tf.nn.lrn(conv2, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75, name="lrn2")
    pool2 = tf.nn.max_pool(lrn2, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1],
                           padding="VALID", name="pool2")
    # 打印池化层的网络结构
    print(pool2.op.name, ' ', pool2.get_shape().as_list())

    # 在命名空间conv3下实现第三个卷积层
    with tf.name_scope("conv3"):
        kernel = tf.Variable(tf.truncated_normal([3, 3, 256, 384],
                                                 dtype=tf.float32, stddev=1e-1),
                             name="weights")
        conv = tf.nn.conv2d(pool2, kernel, [1, 1, 1, 1], padding="SAME")
        biases = tf.Variable(tf.constant(0.0, shape=[384], dtype=tf.float32),
                             trainable=True, name="biases")
        conv3 = tf.nn.relu(tf.nn.bias_add(conv, biases))
        parameters += [kernel, biases]

        # 打印第三个卷积层的网络结构
        print(conv3.op.name, ' ', conv3.get_shape().as_list())

    # 在命名空间conv4下实现第四个卷积层
    with tf.name_scope("conv4"):
        kernel = tf.Variable(tf.truncated_normal([3, 3, 384, 384],
                                                 dtype=tf.float32, stddev=1e-1),
                             name="weights")
        conv = tf.nn.conv2d(conv3, kernel, [1, 1, 1, 1], padding="SAME")
        biases = tf.Variable(tf.constant(0.0, shape=[384], dtype=tf.float32),
                             trainable=True, name="biases")
        conv4 = tf.nn.relu(tf.nn.bias_add(conv, biases))
        parameters += [kernel, biases]

        # 打印第四个卷积层的网络结构
        print(conv4.op.name, ' ', conv4.get_shape().as_list())

    # 在命名空间conv5下实现第五个卷积层
    with tf.name_scope("conv5"):
        kernel = tf.Variable(tf.truncated_normal([3, 3, 384, 256],
                                                 dtype=tf.float32, stddev=1e-1),
                             name="weights")
        conv = tf.nn.conv2d(conv4, kernel, [1, 1, 1, 1], padding="SAME")
        biases = tf.Variable(tf.constant(0.0, shape=[256], dtype=tf.float32),
                             trainable=True, name="biases")

        conv5 = tf.nn.relu(tf.nn.bias_add(conv, biases))
        parameters += [kernel, biases]

        # 打印第五个卷积层的网络结构
        print(conv5.op.name, ' ', conv5.get_shape().as_list())

    # 添加一个最大池化层
    pool5 = tf.nn.max_pool(conv5, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1],
                           padding="VALID", name="pool5")
    # 打印最大池化层的网络结构
    print(pool5.op.name, ' ', pool5.get_shape().as_list())

    # 将pool5输出的矩阵汇总为向量的形式，为的是方便作为全连层的输入
    pool_shape = pool5.get_shape().as_list()
    nodes = pool_shape[1] * pool_shape[2] * pool_shape[3]
    reshaped = tf.reshape(pool5, [pool_shape[0], nodes])

    # 创建第一个全连接层
    with tf.name_scope("fc_1"):
        fc1_weights = tf.Variable(tf.truncated_normal([nodes, 4096], dtype=tf.float32,
                                                      stddev=1e-1), name="weights")
        fc1_bias = tf.Variable(tf.constant(0.0, shape=[4096],
                                           dtype=tf.float32), trainable=True, name="biases")
        fc_1 = tf.nn.relu(tf.matmul(reshaped, fc1_weights) + fc1_bias)
        parameters += [fc1_weights, fc1_bias]

        # 打印第一个全连接层的网络结构信息
        print(fc_1.op.name, ' ', fc_1.get_shape().as_list())

    # 创建第二个全连接层
    with tf.name_scope("fc_2"):
        fc2_weights = tf.Variable(tf.truncated_normal([4096, 4096], dtype=tf.float32,
                                                      stddev=1e-1), name="weights")
        fc2_bias = tf.Variable(tf.constant(0.0, shape=[4096],
                                           dtype=tf.float32), trainable=True, name="biases")
        fc_2 = tf.nn.relu(tf.matmul(fc_1, fc2_weights) + fc2_bias)
        parameters += [fc2_weights, fc2_bias]

        # 打印第二个全连接层的网络结构信息
        print(fc_2.op.name, ' ', fc_2.get_shape().as_list())

    # 返回全连接层处理的结果
    return fc_2, parameters


with tf.Graph().as_default():
    # 创建模拟的图片数据.
    image_size = 224
    images = tf.Variable(tf.random_normal([batch_size, image_size, image_size, 3],
                                          dtype=tf.float32, stddev=1e-1))

    # 在计算图中定义前向传播模型的运行，并得到不包括全连部分的参数
    # 这些参数用于之后的梯度计算
    fc_2, parameters = inference_op(images)

    init_op = tf.global_variables_initializer()

    # 配置会话，gpu_options.allocator_type 用于设置GPU的分配策略，值为"BFC"表示
    # 采用最佳适配合并算法
    config = tf.ConfigProto()
    config.gpu_options.allocator_type = "BFC"
    with tf.Session(config=config) as sess:
        sess.run(init_op)

        num_steps_burn_in = 10
        total_dura = 0.0
        total_dura_squared = 0.0

        back_total_dura = 0.0
        back_total_dura_squared = 0.0

        for i in range(num_batches + num_steps_burn_in):

            start_time = time.time()
            _ = sess.run(fc_2)
            duration = time.time() - start_time
            if i >= num_steps_burn_in:
                if i % 10 == 0:
                    print('%s: step %d, duration = %.3f' %
                          (datetime.now(), i - num_steps_burn_in, duration))
                total_dura += duration
                total_dura_squared += duration * duration
        average_time = total_dura / num_batches

        # 打印前向传播的运算时间信息
        print('%s: Forward across %d steps, %.3f +/- %.3f sec / batch' %
              (datetime.now(), num_batches, average_time,
               math.sqrt(total_dura_squared / num_batches - average_time * average_time)))

        # 使用gradients()求相对于pool5的L2 loss的所有模型参数的梯度
        # 函数原型gradients(ys,xs,grad_ys,name,colocate_gradients_with_ops,gate_gradients,
        # aggregation_method=None)
        # 一般情况下我们只需对参数ys、xs传递参数，他会计算ys相对于xs的偏导数，并将
        # 结果作为一个长度为len(xs)的列表返回，其他参数在函数定义时都带有默认值，
        # 比如grad_ys默认为None，name默认为gradients，colocate_gradients_with_ops默认
        # 为False，gate_gradients默认为False
        grad = tf.gradients(tf.nn.l2_loss(fc_2), parameters)

        # 运行反向传播测试过程
        for i in range(num_batches + num_steps_burn_in):
            start_time = time.time()
            _ = sess.run(grad)
            duration = time.time() - start_time
            if i >= num_steps_burn_in:
                if i % 10 == 0:
                    print('%s: step %d, duration = %.3f' %
                          (datetime.now(), i - num_steps_burn_in, duration))
                back_total_dura += duration
                back_total_dura_squared += duration * duration
        back_avg_t = back_total_dura / num_batches

        # 打印反向传播的运算时间信息
        print('%s: Forward-backward across %d steps, %.3f +/- %.3f sec / batch' %
              (datetime.now(), num_batches, back_avg_t,
               math.sqrt(back_total_dura_squared / num_batches - back_avg_t * back_avg_t)))

---

2021/9/9