初识机器学习 | 5.梯度下降

import numpy as np
import matplotlib.pyplot as plt

%matplotlib
%matplotlib inline
%config InlineBackend.figure_format = 'retina'

Using matplotlib backend: MacOSX

梯度下降求解一元二次方程

$$y=(x-2.5{)}^2-1$$

x = np.linspace(-1,6,200)
y = (x - 2.5)**2 - 1

plt.plot(x, y)
plt.show()

def j(theta):
    """一元二次方程"""
    try:
        return (theta - 2.5) ** 2 - 1
    except:
        return float('inf')

def dj(theta):
    """求导"""
    return 2 * (theta - 2.5)

def gradient_descent(theta=0.0, eta=0.01, epsilon=1e-8, max_iters=10000):
    """
    theta: 参数
    eta: 学习率
    epsilon: 最小值
    max_iters: 最大尝试次数
    """
    theta_history = [theta]
    while max_iters>0:
        gradient = dj(theta)
        last_theta = theta
        theta = theta - eta * gradient
        theta_history.append(theta)
        
        if(abs(j(theta)-j(last_theta)) < epsilon):
            break
        max_iters -= 1
    print('theta: ', theta)
    print('min j(theta): ', j(theta))
    print('theta_history length: ', len(theta_history))
    plt.plot(x, y)
    plt.plot(np.array(theta_history), j(np.array(theta_history)), color="r", marker='+')
    plt.show()
    

# 使用默认参数
gradient_descent()

theta:  2.4995140741236224
min j(theta):  -0.9999997638760426
theta_history length:  424

# 当学习率eta比较小时，下降的步子很小。程序需要循环很多次才会找到最小值。 一般学习率会设置为0.01
gradient_descent(eta=0.001)

theta:  2.4984243400819484
min j(theta):  -0.9999975172958226
theta_history length:  3682

# 当学习率eta较大时候, theta跳到右边
gradient_descent(eta=0.8)

theta:  2.500054842376601
min j(theta):  -0.9999999969923137
theta_history length:  22

# 当特别大时，如1.5. 程序会陷入死循环, 所以限定递归次数为500
gradient_descent(eta=1.5, max_iters=500)

theta:  -8.183476519740352e+150
min j(theta):  6.696928794914166e+301
theta_history length:  501

线性回归模型中使用梯度下降

损失函数:

$$J(\theta )=\frac{1}{m}\sum _{i=1}^m{\left(y^{(i)}-{\hat{y}}^{(i)}\right)}^2$$

$$\nabla J(\mathbf{\theta })=\left(\begin{array}{c}\partial J/{\partial }_{{\theta }_0}\\ \partial J/\partial {\theta }_1\\ \partial J/{\partial }_{{\theta }_2}\\ \dots \\ \partial J_{\partial {\theta }_n}\end{array}\right)=\frac{2}{m}\cdot (\begin{array}{c}{\sum }_{i=1}^m\left(X_b^{(i)}\theta -y^{(0)}\right)\\ {\sum }_{i=1}^m\left(X_b^{(i)}\theta -y^{(i)}\right)\cdot X_1^{(i)}\\ {\sum }_{i=1}^m\left(X_b^{(i)}\theta -y^{(i)}\right)\cdot X_2^{(i)}\\ \cdots \\ {\sum }_{i=1}^m(X_b^{(i)}\theta -y^{}\end{array}$$