为了深入理解Gradient Descent算法,写了如下代码。
在y = 2x直线上生成随机高斯噪声
# -*- coding: utf-8 -*-
"""
Created on Wed Aug 30 15:27:51 2017
@author: liuxy
"""
import numpy as np
import matplotlib.pyplot as plt
def gen_data(size):
x = np.arange(0, size, 1)
e = np.random.normal(0, 3, size)
y = 2*x + e
return [x, y]
def compute_gradient_full(data, w):
X = data[0]
Y = data[1]
N = len(X)
g = np.sum(2*X*(X*w - Y))/N
return g
def compute_gradient_SGD(data, w):
X = data[0]
Y = data[1]
idx = np.random.randint(0, len(X)-1)
d = X[idx]
t = Y[idx]
g = 2*d*(d*w - t)
return g
def compute_gradient_miniBatch(data, w):
X = data[0]
Y = data[1]
N = 16
X_b = []
Y_b = []
for i in range(N):
idx = np.random.randint(0, len(X)-1)
X_b.append(X[idx])
Y_b.append(Y[idx])
X_ba = np.array(X_b)
Y_ba = np.array(Y_b)
g = np.sum(2*X_ba*(X_ba*w - Y_ba))/N
return g
def Optimizer(data, w, learning_rate, num_iterator, method, Wts):
for i in range(num_iterator):
g = 0
if ('full' == method):
g = compute_gradient_full(data, w)
if ('mini' == method):
g = compute_gradient_miniBatch(data, w)
if ('sgd' == method):
g = compute_gradient_SGD(data, w)
w = w - learning_rate * g
Wts.append(w)
data = gen_data(100)
#plt.scatter(data[0], data[1])
lr = 0.000020
w = 6
num = 100
Weights_full = []
Weights_mini = []
Weights_sgd = []
Weights_full.append(w)
Weights_mini.append(w)
Weights_sgd.append(w)
Optimizer(data, w, lr, num, 'full', Weights_full)
Optimizer(data, w, lr, num, 'mini', Weights_mini)
Optimizer(data, w, lr, num, 'sgd', Weights_sgd)
plt.plot(np.arange(0,num+1), Weights_full)
plt.plot(np.arange(0,num+1), Weights_mini)
plt.plot(np.arange(0,num+1), Weights_sgd)
权重变化, full, mini batch, sgd