2.1 numpy 实现梯度下降法
import numpy as np
N, D_in, H, D_out = 64, 1000, 100, 5
np.random.seed(2019)
x = np.random.randn(N, D_in)
y = np.random.randn(N, D_out)
w1 = np.random.randn(D_in, H)
w2 = np.random.randn(H, D_out)
learning_rate = 1e-6
for i in range(300):
h = x.dot(w1)
h_relu = np.maximum(h, 0)
y_pred = h_relu.dot(w2)
loss = np.square(y_pred - y).sum()
print("第{}次loss为{}".format(i, loss))
grad_y_pred = 2.0 * (y_pred - y)
grad_w2 = h_relu.T.dot(grad_y_pred)
grad_h_relu = grad_y_pred.dot(w2.T)
grad_h = grad_h_relu.copy()
grad_h[h < 0] = 0
grad_w1 = x.T.dot(grad_h)
w1 -= learning_rate * grad_w1
w2 -= learning_rate * grad_w2
2.2 pytorch 实现梯度下降法
import torch
from torch.autograd import Variable
"""
PyTorch中,所有神经网络的核心是 autograd 包。先简单介绍一下这个包,然后训练我们的第一个的神经网络。
autograd 包为张量上的所有操作提供了自动求导机制。它是一个在运行时定义(define-by-run)的框架,
这意味着反向传播是根据代码如何运行来决定的,并且每次迭代可以是不同的.
Variable讲解
https://www.cnblogs.com/hellcat/p/8449031.html
"""
x = torch.Tensor([1])
x = Variable(x, requires_grad=True)
print('grad', x.grad, 'data', x.data)
learning_rate = 0.1
epochs = 20
for epoch in range(epochs):
y = x ** 2 - 1
y.backward()
print('grad', x.grad.data)
x.data = x.data - learning_rate * x.grad.data
x.grad.data.zero_()
print(x.data)
print(y)