Numpy实现简单的三层神经网络

输入X（60,1000），中间隐藏层的维度分别为：200,100,10

import numpy as np
N,D_in,H_1,H_2,D_out=60,1000,200,100,10
x=np.random.randn(N,D_in) #(60,1000)
y=np.random.rand(N,D_out)#(60,10)

w1=np.random.randn(D_in,H_1)#(1000,200)
w2=np.random.randn(H_1,H_2)#(200,100)
w3=np.random.randn(H_2,D_out)#(100,10)
learning_rate=1e-4
for i in range(500):
    #forward
    h1=x.dot(w1)#(60,200)
    h1_sigmod=1.0/(1+np.exp(-h1))
    h2=h1_sigmod.dot(w2)#(60,100)
    h2_sigmod=1.0/(1+np.exp(-h2))
    y_pred=h2_sigmod.dot(w3)#(60,10)
    #loss
    loss=np.square(y-y_pred).sum()#(60,10)
    print(i,loss)
    #back forward
    grad_y_pred=2.0*(y-y_pred)#(60,10)
    grad_h2_sigmod=grad_y_pred.dot(w3.T)#((60,100)
    grad_w3=h2_sigmod.T.dot(grad_y_pred)#(100,10)
    grad_h2=grad_h2_sigmod*(1-h2_sigmod)*grad_h2_sigmod#(60,100)
    grad_w2=h1_sigmod.T.dot(grad_h2)#(200,100)
    grad_h1_sigmod=grad_h2.dot(w2.T)
    grad_h1=grad_h1_sigmod*(1-h1_sigmod)*h1_sigmod
    grad_w1=x.T.dot(grad_h1)#(1000,200)
    #gradient descent
    w1-=learning_rate*grad_w1
    w2-=learning_rate*grad_w2
    w3-=learning_rate*grad_w3

用pytorch实现可以说是非常方便了

import torch
N,D_in,H_1,H_2,D_out=60,1000,200,100,10
x=torch.randn(N,D_in) #(60,1000)
y=torch.rand(N,D_out)#(60,10)
model=torch.nn.Sequential(
    torch.nn.Linear(D_in,H_1),
    torch.nn.Sigmoid(),
    torch.nn.Linear(H_1,H_2),
    torch.nn.Sigmoid(),
    torch.nn.Linear(H_2,D_out),
)
loss_func=torch.nn.MSELoss(reduction="sum")
learning_rate=1e-4
for i in range(500):
    y_pred=model(x)
    loss=loss_func(y,y_pred)
    print(i,loss.item())
    loss.backward()
    model.zero_grad()
    with torch.no_grad():
        for param in model.parameters():
            param-=learning_rate*param.grad
   
    

用pytorch自带的优化器进一步简化

import torch
N,D_in,H_1,H_2,D_out=60,1000,200,100,10
x=torch.randn(N,D_in) #(60,1000)
y=torch.rand(N,D_out)#(60,10)
model=torch.nn.Sequential(
    torch.nn.Linear(D_in,H_1),
    torch.nn.Sigmoid(),
    torch.nn.Linear(H_1,H_2),
    torch.nn.Sigmoid(),
    torch.nn.Linear(H_2,D_out),
)
loss_func=torch.nn.MSELoss(reduction="sum")
learning_rate=1e-4
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
for i in range(500):
    y_pred=model(x)
    loss=loss_func(y,y_pred)
    print(i,loss.item())
    optimizer.zero_grad
    loss.backward()
    optimizer.step()