tensorflow2.0学习笔记：应用tensorflow近似求导

使用tensorflow近似求导，以及与Optimizer的结合使用，tensorflow2.0学习笔记：自定义求导介绍在神经网络训练中实现自定义求导。

import matplotlib as mpl
import matplotlib.pyplot as plt
%matplotlib inline
import numpy as np
import sklearn
import pandas as pd
import os
import sys
import time
import tensorflow as tf

from tensorflow import keras
print(tf.__version__)

2.0.0

一元多项式，近似求导

def f(x):
    return 3. * x **2 + 2. * x -1

def approximate_derivative(f,x,eps=1e-3):
    return (f(x+eps)-f(x-eps))/(2.*eps)

print(approximate_derivative(f,1.0)) # 在x=1处的真实导数=8

7.999999999999119

二元多项式，近似求导

def g(x1,x2):
    return (x1+5)*(x2**2)

def approximate_gradient(g,x1,x2,eps=1e-3):
    dg_x1 = approximate_derivative(lambda x : g(x,x2),x1,eps) # x1的偏导，固定x2,用lambda表达式
    dg_x2 = approximate_derivative(lambda x : g(x1,x),x2,eps)
    return dg_x1,dg_x2

print(approximate_gradient(g,2.,3.)) # 在(2,3)处的真实导数(9,42).

(8.999999999993236, 41.999999999994486)

使用tensorflow自定义求导

x1 = tf.Variable(2.0)
x2 = tf.Variable(3.0)

# 打开tf.GradientTape
with tf.GradientTape() as tape:
    z = g(x1,x2)

dz_x1 = tape.gradient(z,x1) # 传入两个参数，第一个为函数的输出，第二个为要求导的变量(对哪个变量求偏导)
print(dz_x1)

# tape默认只能用一次，所以不能连续再对x2求偏导
try:
    dz_x2 = tape.gradient(z,x2)
except RuntimeError as ex:
    print(ex)

tf.Tensor(9.0, shape=(), dtype=float32)
GradientTape.gradient can only be called once on non-persistent tapes.

x1 = tf.Variable(2.0)
x2 = tf.Variable(3.0)

# tape打开可保存选项，可以调用多次，但是用完后得自己关闭。
with tf.GradientTape(persistent=True) as tape:
    z = g(x1,x2)

dz_x1 = tape.gradient(z,x1)
print(dz_x1)

dz_x2 = tape.gradient(z,x2)
print(dz_x2)

del tape

tf.Tensor(9.0, shape=(), dtype=float32)
tf.Tensor(42.0, shape=(), dtype=float32)

# 同时求出两个偏导
x1 = tf.Variable(2.0)
x2 = tf.Variable(3.0)
with tf.GradientTape() as tape:
    z = g(x1,x2)
#对两个变量求偏导
dz_x1x2 = tape.gradient(z,[x1,x2])
print(dz_x1x2)

[<tf.Tensor: id=118, shape=(), dtype=float32, numpy=9.0>, <tf.Tensor: id=124, shape=(), dtype=float32, numpy=42.0>]

# 对常量求偏导
x1 = tf.constant(2.0)
x2 = tf.constant(3.0)
with tf.GradientTape() as tape:
    z = g(x1,x2)
# 对常量直接求偏导，返回None
dz_x1x2 = tape.gradient(z,[x1,x2])
print(dz_x1x2)

[None, None]

x1 = tf.constant(2.0)
x2 = tf.constant(3.0)
#对常量求偏导，得通过使用watch关注
with tf.GradientTape() as tape:
    tape.watch(x1)
    tape.watch(x2)
    z = g(x1,x2)
dz_x1x2 = tape.gradient(z,[x1,x2])
print(dz_x1x2)

[<tf.Tensor: id=133, shape=(), dtype=float32, numpy=9.0>, <tf.Tensor: id=139, shape=(), dtype=float32, numpy=42.0>]

多个目标函数，对一个变量求导

x = tf.Variable(5.0)
with tf.GradientTape() as tape:
    z1 = 3*x
    z2 = x**2
tape.gradient([z1,z2],x)

<tf.Tensor: id=169, shape=(), dtype=float32, numpy=13.0>

# 求解二阶导数
x1 = tf.Variable(2.0)
x2 = tf.Variable(3.0)
# 二阶导数的实现，需要进行嵌套
with tf.GradientTape(persistent=True) as outer_tape:
    with tf.GradientTape(persistent=True) as inner_tape:
        z = g(x1,x2)
    inner_grads = inner_tape.gradient(z,[x1,x2])
    print(inner_grads)
outer_grads = [outer_tape.gradient(inner_grad,[x1,x2])
              for inner_grad in inner_grads]

print(outer_grads)
del inner_tape
del outer_tape

[<tf.Tensor: id=162, shape=(), dtype=float32, numpy=9.0>, <tf.Tensor: id=168, shape=(), dtype=float32, numpy=42.0>]
[[None, <tf.Tensor: id=176, shape=(), dtype=float32, numpy=6.0>], [<tf.Tensor: id=187, shape=(), dtype=float32, numpy=6.0>, <tf.Tensor: id=185, shape=(), dtype=float32, numpy=14.0>]]

模拟梯度下降法

learning_rate = 0.1
x = tf.Variable(0.0)
# 梯度下降：(1)获得导数，(2)更新
for _ in range(50):
    with tf.GradientTape() as tape:
        z = f(x)
    dz_dx = tape.gradient(z,x)
    x.assign_sub(learning_rate * dz_dx) #-=更新
print(x) 

# f(x)在x=-1/3处取得最小值，输出为-0.3333很接近。

<tf.Variable 'Variable:0' shape=() dtype=float32, numpy=-0.3333333>

与Optimizer的结合使用

learning_rate = 0.1
x = tf.Variable(0.0)

optimizer = keras.optimizers.SGD(lr = learning_rate)
for _ in range(10):
    with tf.GradientTape() as tape:
        z = f(x)
    dz_dx = tape.gradient(z,x)
    print(dz_dx)
    optimizer.apply_gradients([(dz_dx,x)]) #更新导数，传入参数是列表
print(x)

tf.Tensor(2.0, shape=(), dtype=float32)
tf.Tensor(0.79999995, shape=(), dtype=float32)
tf.Tensor(0.31999993, shape=(), dtype=float32)
tf.Tensor(0.12800002, shape=(), dtype=float32)
tf.Tensor(0.051199913, shape=(), dtype=float32)
tf.Tensor(0.020480037, shape=(), dtype=float32)
tf.Tensor(0.008191943, shape=(), dtype=float32)
tf.Tensor(0.0032767057, shape=(), dtype=float32)
tf.Tensor(0.0013105869, shape=(), dtype=float32)
tf.Tensor(0.00052428246, shape=(), dtype=float32)
<tf.Variable 'Variable:0' shape=() dtype=float32, numpy=-0.3332984>