【机器学习】梯度下降法与线性回归

基础

参考：机器学习训练营，梯度下降算法的设计与实现，讲师版_哔哩哔哩_bilibili

求极小值

参考：看动画，成为机器学习工程师，梯度下降算法的设计与实现_哔哩哔哩_bilibili

# 计算x位置的梯度
def gradient(x):
    return 2.0 * x - 4  # 函数f(x)=x^2-4x-5的梯度


# 梯度下降的过程中，函数返回迭代完成后，f(x)取的最小值时的x值
def gradient_descent():
    x = 0.0  # 从位置0开始迭代
    iteration_num = 5000  # 迭代次数
    alpha = 0.001  # 迭代速率
    for i in range(0, iteration_num):
        print("%d iteration x=%lf gradient(x)=%lf" % (i, x, gradient(x)))
        x = x - alpha * gradient(x)  # 进行梯度下降，修改x的值
    return x


if __name__ == '__main__':
    gradient_descent()

0 iteration x=0.000000 gradient(x)=-4.000000
1 iteration x=0.004000 gradient(x)=-3.992000
2 iteration x=0.007992 gradient(x)=-3.984016
3 iteration x=0.011976 gradient(x)=-3.976048
4 iteration x=0.015952 gradient(x)=-3.968096
5 iteration x=0.019920 gradient(x)=-3.960160
...
4994 iteration x=1.999909 gradient(x)=-0.000182
4995 iteration x=1.999909 gradient(x)=-0.000182
4996 iteration x=1.999909 gradient(x)=-0.000181
4997 iteration x=1.999910 gradient(x)=-0.000181
4998 iteration x=1.999910 gradient(x)=-0.000181
4999 iteration x=1.999910 gradient(x)=-0.000180

梯度下降迭代求线性回归

$loss={\sum }_{i=0}^n(w\ast x+b-y_i{)}^2$

$w^{{}^{\prime }}=w-lr\ast \frac{\partial loss}{\partial w}$

$b^{{}^{\prime }}=w-lr\ast \frac{\partial loss}{\partial b}$

$y^{{}^{\prime }}=w^{{}^{\prime }}\ast x+b^{{}^{\prime }}$

$\frac{\partial loss}{\partial w}=2{\sum }_{i=0}^n(w\ast x+b-y_i)\ast x$

$\frac{\partial loss}{\partial b}=2{\sum }_{i=0}^n(w\ast x+b-y_i)$

data.csv是存有100个（x，y）点的数据
github

import numpy as np

# 原来的函数
# data = []
# for i in range(100):
# 	x = np.random.uniform(3., 12.)
# 	# mean=0, std=0.1
# 	eps = np.random.normal(0., 0.1)
# 	y = 1.477 * x + 0.089 + eps
# 	data.append([x, y])
# data = np.array(data)
# print(data.shape, data)

# 返回loss
def compute_error_for_line_given_points(b, w, points):
    totalError = 0
    for i in range(0, len(points)):
        x = points[i, 0]  # 相当于points[i][0]
        y = points[i, 1]
        # computer mean-squared-error
        totalError += (y - (w * x + b)) ** 2 # loss
    # average loss for each point
    return totalError / float(len(points))

# 返回经过一次梯度下降迭代后更新的w，b
def step_gradient(b_current, w_current, points, learningRate):
    w_gradient = 0  # 导数dloss/dw
    b_gradient = 0  # 导数dloss/db
    N = float(len(points))
    for i in range(0, len(points)):
        x = points[i, 0]
        y = points[i, 1]
        # grad_w = 2(wx+b-y)*x
        w_gradient += (2 / N) * x * ((w_current * x + b_current) - y)  # 这里 /N 是为了减少计算量，因为求和后的数比较大
        # grad_b = 2(wx+b-y)
        b_gradient += (2 / N) * ((w_current * x + b_current) - y)

    # update w' b'
    new_w = w_current - (learningRate * w_gradient)
    new_b = b_current - (learningRate * b_gradient)
    return [new_b, new_w]

# 进行迭代
def gradient_descent_runner(points, starting_b, starting_w, learning_rate, num_iterations):
    b = starting_b
    w = starting_w
    # update for several times
    for i in range(num_iterations):
        b, w = step_gradient(b, w, np.array(points), learning_rate)
    return [b, w]


def run():
    points = np.genfromtxt("data.csv", delimiter=",")
    learning_rate = 0.0001
    initial_b = 0  # initial y-intercept guess
    initial_w = 0  # initial slope guess
    num_iterations = 1000 # 迭代次数
    print("Starting gradient descent at b = {0}, w = {1}, error = {2}"
          .format(initial_b, initial_w,
                  compute_error_for_line_given_points(initial_b, initial_w, points))
          )
    print("Running...")
    [b, w] = gradient_descent_runner(points, initial_b, initial_w, learning_rate, num_iterations)
    print("After {0} iterations b = {1}, w = {2}, error = {3}".
          format(num_iterations, b, w,
                 compute_error_for_line_given_points(b, w, points))
          )


if __name__ == '__main__':
    run()

Starting gradient descent at b = 0, w = 0, error = 5565.107834483211
Running...
After 1000 iterations b = 0.08893651993741346, w = 1.4777440851894448, error = 112.61481011613473

如果那里不除以N的话会溢出报错