tensorflow2教程-随机梯度下降-反向传播

常见激活函数的梯度

1、ReLU 函数
2、sigmod函数

a = tf.linspace(-10., 10., 10)
with tf.GradientTape() as tape:
	tape.watch(a)  # 如果将a定义为Variable，此句话可以省略
	y = tf.sigmoid(a)
grads = tape.gradient(y, [a])

3、tanh函数
4、leakyReLU函数

损失函数梯度

1、均方差损失函数

with tf.GradientTape() as tape:
	tape.watch([w, b])
	logits = tf.sigmoid(x@w+b) 
	loss = tf.reduce_mean(tf.losses.MSE(y, logits))

grads = tape.gradient(loss, [w, b])
print('w grad:', grads[0])
print('b grad:', grads[1])

2 、交叉熵函数梯度

with tf.GradientTape() as tape:
	tape.watch([w, b])
	logits = (x@w+b)
	loss = tf.reduce_mean(tf.losses.categorical_crossentropy(y, logits, rom_logits=True))
	#rom_logits=True 可以省略softmax那一步
grads = tape.gradient(loss, [w, b])

3、反向传播推倒：略

4、函数优化

def himmelblau(x):
 # himmelblau 函数实现
    return (x[0] ** 2 + x[1] - 11) ** 2 + (x[0] + x[1] ** 2 - 7) ** 2
x = tf.constant([-4.,0.])

for step in range(200):
    with tf.GradientTape() as tape:
        tape.watch(x)
        y = himmelblau(x)
    grads = tape.gradient(y,x)
    x -= 0.01*grads

    if step % 20 == 0:
        print('step{}:x = {},f(x) = {}'.format(step, x, y.numpy()))

minist数据模型

import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import datasets, layers, optimizers, Sequential, metrics
import os

os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'


def preprocess(x, y):
    """
    :param x:
    :param y:
    :return:
    """
    # [b, 28, 28], [b]
    x = tf.cast(x, dtype=tf.float32) / 255.
    y = tf.cast(y, dtype=tf.int32)
    y = tf.one_hot(y, depth=10)
    return x, y



def main():
    (x, y), (x_test, y_test) = datasets.fashion_mnist.load_data()

    db = tf.data.Dataset.from_tensor_slices((x, y))
    db = db.map(preprocess).shuffle(10000).batch(128)

    db_test = tf.data.Dataset.from_tensor_slices((x_test, y_test))
    db_test = db_test.map(preprocess).batch(128)

    #db_iter = iter(db)
    #sample = next(db_iter)
    # print(sample[1].shape)

    model = Sequential([
        layers.Dense(256, activation=tf.nn.relu),
        layers.Dense(128, activation=tf.nn.relu),
        layers.Dense(64, activation=tf.nn.relu),
        layers.Dense(32, activation=tf.nn.relu),
        layers.Dense(10, activation=tf.nn.relu),
    ])
    #model.build(input_shape=[None, 28 * 28])
    #model.summary()
    optimizer = optimizers.Adam(lr=1e-3)
    for epock in range(30):
        for step, (x, y) in enumerate(db):
            x = tf.reshape(x, [-1, 28 * 28])
            with tf.GradientTape() as tape:
                logits = model(x)
                loss_mse = tf.reduce_mean(tf.losses.MSE(y, logits))
                loss_cro = tf.reduce_mean(tf.losses.categorical_crossentropy(y, logits, from_logits=True))
            grads = tape.gradient(loss_mse, model.trainable_variables)
            optimizer.apply_gradients(zip(grads, model.trainable_variables))
            if step % 200 == 0:
                print(epock, step, 'loss', float(loss_mse), float(loss_cro))
        total_num = 0
        total_correct = 0
        for x, y in db_test:
            x = tf.reshape(x, [-1, 28 * 28])
            logit = model(x)
            prob = tf.nn.softmax(logit, axis=1)
            pred = tf.argmax(prob, axis=1)
            pred = tf.cast(pred, tf.int32)
            y = tf.argmax(y, axis=1)
            y = tf.cast(y, dtype=tf.int32)
            correct = tf.equal(pred, y)
            correct = tf.reduce_sum(tf.cast(correct, dtype=tf.int32))
            total_correct += int(correct)
            total_num += x.shape[0]
        acc = total_correct / total_num
        print(acc)


if __name__ == '__main__':
    main()

可视化

命令：tensorboard --logdir log

def plot_to_image(figure):
    """Converts the matplotlib plot specified by 'figure' to a PNG image and
    returns it. The supplied figure is closed and inaccessible after this call."""
    # Save the plot to a PNG in memory.
    buf = io.BytesIO()
    plt.savefig(buf, format='png')
    # Closing the figure prevents it from being displayed directly inside
    # the notebook.
    plt.close(figure)
    buf.seek(0)
    # Convert PNG buffer to TF image
    image = tf.image.decode_png(buf.getvalue(), channels=4)
    # Add the batch dimension
    image = tf.expand_dims(image, 0)
    return image


def image_grid(images):
    """Return a 5x5 grid of the MNIST images as a matplotlib figure."""
    # Create a figure to contain the plot.
    figure = plt.figure(figsize=(10, 10))
    for i in range(16):
        # Start next subplot.
        plt.subplot(4, 4, i + 1, title='name')
        plt.xticks([])
        plt.yticks([])
        plt.grid(False)
        plt.imshow(images[i])
    return figure
    
with summary_writer.as_default():
    tf.summary.scalar('test-acc', float(total_correct / total_num), step=step)
    tf.summary.image("val-onebyone-images:", val_images, max_outputs=25, step=step)

    val_images = tf.reshape(val_images, [-1, 28, 28])

    figure = image_grid(val_images)
    tf.summary.image('val-images:', plot_to_image(figure), step=step)

    tf.summary.scalar('loss', float(loss_mse), step=epock)
    tf.summary.scalar('acc', float(total_correct / total_num), step=epock)