BP算法（误差逆传播算法）简单实现

1️⃣要求

设计一个5层的全连接网络，损失函数自由，激励函数使用sigmoid/tanh/relu，反向传播过程自己写，不能使用pytorch框架的自动求导机制，目标是实现如下函数：

2️⃣代码

import numpy as np
import matplotlib.pyplot as plt

x = np.arange(-7, 5, 0.05)
y = x ** 2 + 2 * x - 3
x = x.reshape(-1, 1)
y = y.reshape(-1, 1)
N, D_in, H1, H2, H3, D_out = x.shape[0], 1, 100, 100, 100, 1

w1 = 0.2*np.random.randn(H1, D_in)
w2 = 0.2*np.random.randn(H2, H1)
w3 = 0.2*np.random.randn(H3, H2)
w4 = 0.2*np.random.randn(D_out, H3)

b1 = np.random.randn(H1, 1)
b2 = np.random.randn(H2, 1)
b3 = np.random.randn(H3, 1)
b4 = np.random.randn(D_out, 1)

learning_rate = 1e-5

x=x.T
y=y.T

def ReLu_dev(x):
    # ReLu 导数
    y = np.array(x, copy=True)
    y[x > 0] = 1.
    y[x <= 0] = 0.
    return y
for it in range(2000):

    # forward pass
    Hlayer1 = w1.dot(x)+b1
    Hlayer1_A = np.maximum(Hlayer1, 0)

    Hlayer2 = w2.dot(Hlayer1_A)+b2
    Hlayer2_A = np.maximum(Hlayer2, 0)

    Hlayer3 = w3.dot(Hlayer2_A)+b3
    Hlayer3_A = np.maximum(Hlayer3, 0)

    y_pred = w4.dot(Hlayer3_A)+b4

    # compute loss
    loss = 0.5*np.square(y_pred-y).sum()
    print(it, loss)

    error_y_pred = (y_pred-y)
    grad_w4 = np.dot(error_y_pred,Hlayer3_A.transpose())

    error_hlayer3 = np.dot(w4.transpose(), error_y_pred) * ReLu_dev(Hlayer3)
    grad_w3 = np.dot(error_hlayer3, Hlayer2_A.transpose())

    error_hlayer2 = np.dot(w3.transpose(), error_hlayer3) * ReLu_dev(Hlayer2)
    grad_w2 = np.dot(error_hlayer2, Hlayer1_A.transpose())

    error_hlayer1 = np.dot(w2.transpose(), error_hlayer2) * ReLu_dev(Hlayer1)
    grad_w1 = np.dot(error_hlayer1, x.transpose())

    grad_b4 = np.dot(error_y_pred, np.ones((N, 1)))
    grad_b3 = np.dot(error_hlayer3, np.ones((N, 1)))
    grad_b2 = np.dot(error_hlayer2, np.ones((N, 1)))
    grad_b1 = np.dot(error_hlayer1, np.ones((N, 1)))

    # update
    w1 -= learning_rate * grad_w1
    w2 -= learning_rate * grad_w2
    w3 -= learning_rate * grad_w3
    w4 -= learning_rate * grad_w4
    b1 -= learning_rate * grad_b1
    b2 -= learning_rate * grad_b2
    b3 -= learning_rate * grad_b3
    b4 -= learning_rate * grad_b4

# show figure
plt.scatter(x, y)

Hlayer1 = w1.dot(x)+b1
Hlayer1_A = np.maximum(Hlayer1, 0)

Hlayer2 = w2.dot(Hlayer1_A)+b2
Hlayer2_A = np.maximum(Hlayer2, 0)

Hlayer3 = w3.dot(Hlayer2_A)+b3
Hlayer3_A = np.maximum(Hlayer3, 0)

y_pred = w4.dot(Hlayer3_A)+b4

plt.scatter(x, y_pred)
plt.show()

3️⃣可视化

 

人生能有几回搏？！