TensorFlow 2.0 笔记（四）—— 随机梯度下降

导数
偏微分
梯度

#tf.GradientTape
w = tf.constant(1.)
x = tf.constant(2.)
y = x*w

with tf.GradientTape() as tape:
    tape.watch([w])
    y2 = x * w
    
grad1 = tape.gradient(y, [w])

with tf.GradientTape() as tape:
    tape.watch([w])
    y2 = x * w
    
grad2 = tape.gradient(y2, [w])

with tf.GradientTape(persistent=True) as tape:
    tape.watch([w])
    y2 = x * w

grad3 = tape.gradient(y2, [w])
grad4 = tape.gradient(y2, [w])

二阶导数

import tensorflow as tf
w = tf.Variable(1.)
b = tf.Variable(2.)
x = tf.Variable(3.)
with tf.GradientTape() as t1:

    with tf.GradientTape() as t2:
        y = x * w + b
    dy_dw, dy_db = t2.gradient(y, [w, b])

d2y_dw2 = t1.gradient(dy_dw, w)

print(dy_dw)
print(dy_db)
print(d2y_dw2)

assert dy_dw.numpy() == 3.0
assert d2y_dw2 is None

激活函数

Sigmoid / Logistic

$$f(x)=\sigma (x)=\frac{1}{1+e^{-x}}$$
$${\sigma }^{\prime }=\sigma (1-\sigma )$$

a = tf.linspace(-10., 10., 10)
with tf.GradientTape() as tape:
    tape.watch(a)
    y = tf.nn.sigmoid(a)

grads = tape.gradient(y, [a])

Tanh
$$f(x)=tanh(x)=\frac{e^x-e^{-x}}{e^x+e^{-x}}=2sigmoid(2x)-1$$
$$\frac{d}{dx}tanh(x)=1-tan{h}^2(x)$$

a = tf.linspace(-5., 5., 10)
with tf.GradientTape() as tape:
    tape.watch(a)
    y = tf.nn.tanh(a)

grads = tape.gradient(y, [a])

Rectified Linear Unit
$$f(x)=\{\begin{array}{rcl}0& for& x<0\\ & & \\ x& for& x⩾0\end{array}$$
$$f^{\prime }(x)=\{\begin{array}{rcl}0& for& x<0\\ & & \\ 1& for& x⩾0\end{array}$$

a = tf.linspace(-1., 1., 10)
tf.nn.relu(a)
tf.nn.leaky_relu(a)

Typical Loss

• Mean Squared Error
  • $loss=\sum [y-(xw+b){]}^2$
  • $L_{2-norm}=||y-(xw+b)|{|}_2$
  • $loss=norm(y-(xw+b){)}^2$
• MSE Derivative
  • $loss=\sum [y-f_{\theta }(x){]}^2$
  • $\frac{\nabla loss}{\nabla \theta }=2\sum [y-f_{\theta }(x)]\ast \frac{\nabla f_{\theta }(x)}{\nabla \theta }$

x = tf.random.normal([2, 4])
w = tf.random.normal([4, 3])
b = tf.zeros([3])
y = tf.constant([2, 0])

with tf.GradientTape() as tape:
    tape.watch([w, b]) #tf.Variable则不需要
    prob = tf.nn.softmax(x@w+b, axis=1)
    loss = tf.reduce_mean(tf.losses.MSE(tf.one_hot(y, depth=3), prob))

grads = tape.gradient(loss, [w, b])
grads[0]
grads[1]

• Cross Entropy Loss
  • binary
  • multi-class
  • softmax
    • soft version of max
      $$S(y_i)=\frac{e^{y_i}}{{\sum }_j{e}^{y_i}}$$
  • Leave it to Logistic Regression Part
• SoftMax Derivative
  $$p_i=\frac{e^{a_i}}{{\sum }_{k=1}^N{e}^{a_k}}$$
  $$\frac{\partial p_i}{\partial a_j}=\{\begin{array}{rcl}{p}_i(1-p_j)& if& i=j\\ & & \\ -p_j.p_i& if& i\ne j\end{array}$$
  
  Or using Kronecker delta ${\delta }_{ij}=\{\begin{array}{rcl}1& if& i=j\\ & & \\ 0& if& i\ne j\end{array}$

$$\frac{\partial p_i}{\partial a_j}=p_i({\delta }_{ij}-p_j)$$

x = tf.random.normal([2, 4])
w = tf.random.normal([4, 3])
b = tf.zeros([3])
y = tf.constant([2, 0])

with tf.GradientTape() as tape:
    tape.watch([w, b])
    logits = x@w+b
    loss = tf.reduce_mean(tf.losses.categorical_crossentropy(tf.one_hot(y, depth=3), logits, from_logits=True))

grads = tape.gradient(loss, [w, b])
grads[0]
grads[1]

感知机及其梯度

单输出感知机

Perceptron

• $y=XW+b$
• $y=\sum x_i\ast w_i+b$

Derivative
$$E=\frac{1}{2}(O_0^1-t{)}^2$$
$$\frac{\partial E}{\partial w_{j0}}=(O_0-t)\frac{\partial O_0}{\partial w_{j0}}$$
$$\frac{\partial E}{\partial w_{j0}}=(O_0-t)\frac{\partial \sigma (x_0)}{\partial w_{j0}}$$
$$\frac{\partial E}{\partial w_{j0}}=(O_0-t)\sigma (x_0)(1-\sigma (x_0))\frac{\partial x_0^1}{\partial w_{j0}}$$
$$\frac{\partial E}{\partial w_{j0}}=(O_0-t)O_0(1-O_0)\frac{\partial x_0^1}{\partial w_{j0}}$$
$$\frac{\partial E}{\partial w_{j0}}=(O_0-t)O_0(1-O_0)x_j^0$$

x = tf.random.normal([1, 3])
w = tf.ones([3, 1])
b = tf.ones([1])
y = tf.constant([1])

with tf.GradientTape() as tape:
    tape.watch([w, b])
    prob = tf.nn.sigmoid(x@w+b)
    loss = tf.reduce_mean(tf.losses.MSE(y, prob))

grads = tape.gradient(loss, [w, b])
grads[0]
grads[1]

多输出感知机
Derivative
$$E=\frac{1}{2}\sum (O_i^1-t_i{)}^2$$
$$\frac{\partial E}{\partial w_{jk}}=(O_k-t_k)\frac{\partial O_k}{\partial w_{jk}}$$
$$\frac{\partial E}{\partial w_{jk}}=(O_k-t_k)\frac{\partial \sigma (x_k)}{\partial w_{jk}}$$
$$\frac{\partial E}{\partial w_{jk}}=(O_k-t_k)\sigma (x_k)(1-\sigma (x_k))\frac{\partial x_k^1}{\partial w_{jk}}$$
$$\frac{\partial E}{\partial w_{jk}}=(O_k-t_k)O_k(1-O_k)\frac{\partial x_k^1}{\partial w_{jk}}$$
$$\frac{\partial E}{\partial w_{jk}}=(O_k-t_k)O_k(1-O_k)x_j^0$$

x = tf.random.normal([2, 4])
w = tf.random.normal([4, 3])
b = tf.zeros([3])
y = tf.constant([2, 0])

with tf.GradientTape() as tape:
    tape.watch([w, b])
    prob = tf.nn.softmax(x@w+b, axis=1)
    loss = tf.reduce_mean(tf.losses.MSE(tf.one_hot(y, depth=3), prob))

grads = tape.gradient(loss, [w, b])
grads[0]
grads[1]

链式法则

x = tf.Variable(1.)
w1 = tf.Variable(2.)
b1 = tf.Variable(1.)
w2 = tf.Variable(2.)
b2 = tf.Variable(1.)

with tf.GradientTape(persistent=True) as tape:
    y1 = x * w1 + b1
    y2 = y1 * w2 + b2

dy2_dy1 = tape.gradient(y2, [y1])[0]
dy1_dw1 = tape.gradient(y1, [w1])[0]
dy2_dw1 = tape.gradient(y2, [w1])[0]

多层感知机模型

$$\frac{\partial E}{\partial w_{jk}}=(O_k-t_k)O_k(1-O_k)O_j^J$$
$$\frac{\partial E}{\partial w_{jk}}={\delta }_k^K{O}_j^J$$
$$\frac{\partial E}{\partial w_{ij}}=\frac{\partial }{\partial w_{ij}}\frac{1}{2}\sum _{k\in K}(O_k-t_k{)}^2$$
$$\frac{\partial E}{\partial w_{ij}}=\sum _{k\in K}(O_k-t_k)\frac{\partial }{\partial W_{ij}}{O}_k$$
$$\frac{\partial E}{\partial w_{ij}}=\sum _{k\in K}(O_k-t_k)\frac{\partial }{\partial W_{ij}}\sigma (x_k)$$
$$\frac{\partial E}{\partial w_{ij}}=\sum _{k\in K}(O_k-t_k)O_k(1-O_k)\frac{\partial x_k}{\partial O_j}.\frac{\partial O_j}{\partial W_{ij}}$$
$$\frac{\partial E}{\partial w_{ij}}=\sum _{k\in K}(O_k-t_k)O_k(1-O_k)W_{jk}\frac{\partial O_j}{\partial W_{ij}}$$
$$\frac{\partial E}{\partial w_{ij}}=\frac{\partial O_j}{\partial W_{ij}}\sum _{k\in K}(O_k-t_k)O_k(1-O_k)W_{jk}$$
$$\frac{\partial E}{\partial w_{ij}}=O_j(1-O_j)\frac{\partial x_j}{\partial W_{ij}}\sum _{k\in K}(O_k-t_k)O_k(1-O_k)W_{jk}$$
$$\frac{\partial E}{\partial w_{ij}}=O_j(1-O_j)O_i\sum _{k\in K}(O_k-t_k)O_k(1-O_k)W_{jk}$$
$$\frac{\partial E}{\partial w_{ij}}=O_i{O}_j(1-O_j)\sum _{k\in K}{\delta }_k{W}_{jk}$$

For an output layer node $k\in K$
$$\frac{\partial E}{\partial W_{jk}}=O_j{\delta }_k$$
where$${\delta }_k=O_k(1-O_k)(O_k-t_k)$$

For a hidden layer node $j\in J$
$$\frac{\partial E}{\partial W_{ij}}=O_i{\delta }_j$$
where$${\delta }_j=O_j(1-O_j)\sum _{k\in K}{\delta }_k{W}_{jk}$$

函数优化

import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D


def himmelblau(x):
    return (x[0] ** 2 + x[1] - 11) ** 2 + (x[0] + x[1] ** 2 - 7) ** 2


x = np.arange(-6, 6, 0.1)
y = np.arange(-6, 6, 0.1)
print('x,y range:', x.shape, y.shape)
X, Y = np.meshgrid(x, y)
print('X,Y maps:', X.shape, Y.shape)
Z = himmelblau([X, Y])

fig = plt.figure('himmelblau')
ax = fig.gca(projection='3d')
ax.plot_surface(X, Y, Z)
ax.view_init(60, -30)
ax.set_xlabel('x')
ax.set_ylabel('y')
plt.show()

x = tf.constant([-4., 0.])
for step in range(200):
    with tf.GradientTape() as tape:
        tape.watch(x)
        y = himmelblau(x)
        
    grads = tape.gradient(y, [x])[0]
    x -= 0.01 * grads
    
    if step % 20 == 0:
        print('step {}: x = {}, f(x) = {}'.format(step, x.numpy(), y.numpy()))

import os
import tensorflow as tf
import tensorflow_datasets as tfds

os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'

dataset, metadata = tfds.load('fashion_mnist', as_supervised=True, with_info=True)
train_dataset, test_dataset = dataset['train'], dataset['test']


def normalize(images, labels):
    images = tf.cast(images, tf.float32)
    images /= 255
    return images, labels


print("datasets", train_dataset.map(normalize))
train_dataset = train_dataset.map(normalize)
test_dataset = test_dataset.map(normalize)

num_train_examples = metadata.splits['train'].num_examples
num_test_examples = metadata.splits['test'].num_examples

BATCH_SIZE = 100
train_dataset = train_dataset.repeat().shuffle(num_train_examples).batch(BATCH_SIZE)
test_dataset = test_dataset.batch(BATCH_SIZE)

model = tf.keras.Sequential([
    tf.keras.layers.Dense(256, activation=tf.nn.relu),
    tf.keras.layers.Dense(128, activation=tf.nn.relu),
    tf.keras.layers.Dense(64, activation=tf.nn.relu),
    tf.keras.layers.Dense(32, activation=tf.nn.relu),
    tf.keras.layers.Dense(10)
])

model.build(input_shape=[None, 28*28])
model.summary()
optimizer = tf.keras.optimizers.Adam(lr=1e-3)


def main():
    for epoch in range(30):
        for step, (x,y) in enumerate(train_dataset):
            x = tf.reshape(x, [-1, 28*28])

            with tf.GradientTape() as tape:
                logits = model(x)
                y_ = tf.one_hot(y, depth=10)
                #loss_mse = tf.reduce_mean(tf.losses.MSE(y_, logits))
                loss_ce = tf.reduce_mean(tf.losses.categorical_crossentropy(y_, logits, from_logits=True))

            grads = tape.gradient(loss_ce, model.trainable_variables)
            optimizer.apply_gradients(zip(grads, model.trainable_variables))

            if step % 500 == 0:
                # test
                total_correct = 0
                total_num = 0
                for x, y in test_dataset:
                    x = tf.reshape(x, [-1, 28 * 28])
                    logits = model(x)
                    prob = tf.nn.softmax(logits, axis=1)
                    pred = tf.argmax(prob, axis=1)
                    correct = tf.equal(pred, y)
                    correct = tf.reduce_sum(tf.cast(correct, dtype=tf.int32)).numpy()

                    total_correct += int(correct)
                    total_num += x.shape[0]

                acc = total_correct / total_num
                print(epoch, step, 'loss:', float(loss_ce), 'test acc:', acc)


if __name__ == '__main__':
    main()