梯度下降法代码

全量梯度下降

# 全量梯度下降

import numpy as np

# 创建数据集 X, y
np.random.seed(1)
X = np.random.rand(100, 1)
y = 4 + 3 * X + np.random.randn(100, 1)
X_b = np.c_[np.ones((100, 1)), X]

# 创建超参数
learning_rate = 0.001
n_iterations = 10000

# 1, 初始化θ, W0 ... Wn, 标准正太分布创建W
theta = np.random.rand(2, 1)
# 4, 判断是否收敛, 一般不会去设定阈值, 而是直接采用设置相对大的迭代次数保证可以收敛
for _ in range(n_iterations):
    # 2, 求梯度, 计算gradient
    gradients = X_b.T.dot(X_b.dot(theta) - y)
    # 3, 应用梯度下降法的公式去调整θ值 θt+1 = θt - η * gradient
    theta = theta - learning_rate * gradients

print(theta)

随机梯度下降

# 随机梯度下降

import numpy as np

# 创建数据集 X, y
X = 2 * np.random.rand(100, 1)
y = 4 + 3 * X + np.random.randn(100, 1)
X_b = np.c_[np.ones((100, 1)), X]

# 创建超参数
n_epochs = 10000
m = 100
learning_rate = 0.001

theta = np.random.randn(2, 1)
for epoch in range(n_epochs):
    for _ in range(m):
        random_index = np.random.randint(m)
        xi = X_b[random_index:random_index + 1]
        yi = y[random_index:random_index + 1]
        gradients = xi.T.dot(xi.dot(theta) - yi)
        theta = theta - learning_rate * gradients

print(theta)

小批量梯度下降

# 小批量梯度下降

import numpy as np

X = 2 * np.random.rand(100, 1)
y = 4 + 3 * X + np.random.randn(100, 1)
X_b = np.c_[np.ones((100, 1)), X]

learning_rate = 0.0001
n_epochs = 10000
m = 100
batch_size = 10
num_batches = int(m / batch_size)

theta = np.random.randn(2, 1)
for epoch in range(n_epochs):
    # 在双层for循环之间, 每个轮次开始分批次迭代之前打乱数据索引顺序
    arr = np.arange(len(X_b))
    np.random.shuffle(arr)
    X_b = X_b[arr]
    y = y[arr]
    for i in range(num_batches):
        X_batch = X_b[i * batch_size: i * batch_size + batch_size]
        y_batch = y[i * batch_size: i * batch_size + batch_size]
        gradients = X_batch.T.dot(X_batch.dot(theta) - y_batch)
        theta = theta - learning_rate * gradients

print(theta)