【ML】- 002 线性回归02-梯度下降法python实现

• 构造数据集

def h(x1, x2, x3):
    return 3 + 2 * x1 - 6 * x2 + 12 * x3


x = [[3, 6, 2], [-5, 3, 9], [9, 4, 1], [-9, 3, 5], [7, 3, -2], [-2, -1, 9], [3, 2, 9]]
y = []
for i, j, k in x:
    v = h(i, j, k)
    y.append(v)

print('x:', x)
print('y:', y)

x: [[3, 6, 2], [-5, 3, 9], [9, 4, 1], [-9, 3, 5], [7, 3, -2], [-2, -1, 9], [3, 2, 9]]
y: [-3, 83, 9, 27, -25, 113, 105]

• Batch Gradient Descent

def h_theta(thetas, x):
    return sum(theta * i for theta, i in zip(thetas, x))


def BGD(thetas, X, y, lr=0.001):
    grads = []
    for i in range(len(thetas)):
        grad = sum([(h_theta(thetas, x) - y_i) * x[i] for x, y_i in zip(X, y)])
        thetas[i] = thetas[i] - lr * grad
    return thetas


thetas = [0.1, 0.1, 0.1, 0.1]
epsilon = 0.00001

X = x
x_input = [[1] + x for x in X]
y_input = y

count = 0
while True:
    thetas = BGD(thetas, x_input, y_input)
    error = sum([(h_theta(thetas, x) - y_i) ** 2 for x, y_i in zip(x_input, y_input)]) / 2
    count += 1
    if error < epsilon:
        print('iteration: ', count)
        print('parameters: ', thetas)
        print('error: ', error)
        break

iteration: 9926
parameters: [2.994408629878809, 2.0001475422469874, -5.999107460829956, 12.00051301194443]
error: 9.995191434884599e-06

• Stochastic Gradient Descent

import random


def SGD(thetas, X, y, lr=0.001):
    idx = random.randint(0, len(X)-1)
    x = X[idx]
    y_i = y[idx]
    
    grads = []
    for i in range(len(thetas)):
        grad = (h_theta(thetas, x) - y_i) * x[i]
        thetas[i] = thetas[i] - lr * grad
    return thetas


thetas = [0.1, 0.1, 0.1, 0.1]
epsilon = 0.00001

x_input = [[1] + x for x in X]
y_input = y

count = 0
while True:
    thetas = SGD(thetas, x_input, y_input)
    error = sum([(h_theta(thetas, x) - y_i) ** 2 for x, y_i in zip(x_input, y_input)]) / 2
    count += 1
    if error < epsilon:
        print('iteration: ', count)
        print('parameters: ', thetas)
        print('error: ', error)
        break

iteration: 67876
parameters: [2.9944354351853373, 2.000164063519207, -5.9991719011823275, 12.000533203142787]
error: 9.98125305133155e-06

• mini-batch Stochastic Gradient Descent

import random


def mbSGD(thetas, X, y, lr=0.001, batch_size=5):
    bs_inx = random.sample(range(len(X)), 5)
    X = [X[i] for i in bs_inx]
    y = [y[i] for i in bs_inx]
    
    grads = []
    for i in range(len(thetas)):
        grad = sum([(h_theta(thetas, x) - y_i) * x[i] for x, y_i in zip(X, y)])
        thetas[i] = thetas[i] - lr * grad
    return thetas


thetas = [0.1, 0.1, 0.1, 0.1]
epsilon = 0.00001

x_input = [[1] + x for x in X]
y_input = y

count = 0
while True:
    thetas = mbSGD(thetas, x_input, y_input)
    error = sum([(h_theta(thetas, x) - y_i) ** 2 for x, y_i in zip(x_input, y_input)]) / 2
    count += 1
    if error < epsilon:
        print('iteration: ', count)
        print('parameters: ', thetas)
        print('error: ', error)
        break

iteration: 13768
parameters: [2.994418744371793, 2.0001439350321077, -5.999120200374596, 12.000506505193652]
error: 9.966880303989201e-06