前言
本文为8月8日Pytorch笔记, 用 Numpy 实现 Backpropagation。
一、Numpy 实现 Backpropagation
代码如下:
import numpy as np
import random
def sigmoid(z):
return 1. / (1. + np.exp(-z))
def sigmoid_prime(z):
return sigmoid(z) * (1 - sigmoid(z))
class MLP_np:
def __init__(self, sizes):
'''
:param sizes: [784, 30, 10]
'''
self.sizes = sizes
self.num_layers = len(sizes) - 1
# sizes: [784, 30, 10]
# w: [ch_out, ch_in]
# b: [ch_out]
self.weights = [np.random.randn(ch2, ch1) for ch1, ch2 in zip(sizes[:-1], sizes[1:])] # [784, 30] [30, 10]
# z = wx + b [30, 1]
self.biases = [np.random.randn(ch, 1) for ch in sizes[1:]]
def forward(self, x):
'''
:param x: [784, 1]
:return: [10, 1]
'''
for b, w in zip(self.biases, self.weights):
# [30, 784] @ [784, 1] ==> [30, 1] + [30, 1] ==> [30, 1]
z = np.dot(w, x) + b
# [30, 1]
x = sigmoid(z)
return x
def backward(self, x, y):
'''
:param x: [784, 1]
:param y: [10, 1] ont_hot encoding
:return:
'''
# 1. forward
# 生成列表保存梯度信息
nabla_w = [np.zeros(w.shape) for w in self.weights]
nabla_b = [np.zeros(b.shape) for b in self.biases]
# 保存每层的激活函数
activations = [x]
# 保存每层的 z
zs = []
activation = x
for b, w in zip(self.biases, self.weights):
z = np.dot(w, activation) + b
activation = sigmoid(z)
zs.append(z)
activations.append(activation)
loss = np.power(activations[-1] - y, 2).sum()
# 2. backward
# 2.1 计算输出层的梯度
# [10, 1] * [10, 1] ==> [10, 1]
delta = activations[-1] * (1 - activations[-1]) * (activations[-1] - y)
nabla_b[-1] = delta
# [10, 1] @ [1, 30] ==> [10, 30]
# acativation: [30, 1]
nabla_w[-1] = np.dot(delta, activations[-2].T)
# 2.2 计算隐藏层的梯度
for l in range(2, self.num_layers+1):
l = -l
z = zs[l]
a = activations[l]
# dlta_j
# [10, 30].T @ [10, 1] ==> [30, 1] * [30, 1]
delta = np.dot(self.weights[l+1].T, delta) * a * (1-a)
nabla_b[l] = delta
# [30, 1] @ [784, 1].T ==> [30, 784]
nabla_w[l] = np.dot(delta, activations[l-1].T)
return nabla_w, nabla_b, loss
def train(self, training_data, epochs, batchsz, lr, test_data):
'''
:param training_data: list of (x, y)
:param epochs: 1000
:param batchsz: 10
:param lr: 0.01
:param test_data: list of (x, y)
:return:
'''
if test_data:
n_test = len(test_data)
n = len(training_data)
for j in range(epochs):
random.shuffle(training_data)
mini_batches = [
training_data[k: k+batchsz]
for k in range(0, n, batchsz)
]
# for every batch in current batch
for mini_batch in mini_batches:
loss = self.update_mini_batch(mini_batch, lr)
if test_data:
print("Epoch {0}: {1} / {2}".format(
j, self.evaluate(test_data), n_test
), loss)
else:
print("Epoch {0} complete".format(j))
def update_mini_batch(self, batch, lr):
'''
:param batch: list of (x, y)
:param lr: 0.01
:return:
'''
nabla_w = [np.zeros(w.shape) for w in self.weights]
nabla_b = [np.zeros(b.shape) for b in self.biases]
loss = 0
# for every sample in current batch
for x, y in batch:
# list of every w/b gradient
# [w1, w2, w3]
nabla_w_, nabla_b_, loss_ = self.backward(x, y)
nabla_w =[accu+cur for accu, cur in zip(nabla_w, nabla_w_)]
nabla_b = [accu + cur for accu, cur in zip(nabla_b, nabla_b_)]
loss += loss_
nabla_w = [w / len(batch) for w in nabla_w]
nabla_b = [b / len(batch) for b in nabla_b]
loss = loss / len(batch)
# w = w - lr * nabla_w
self.weights = [w - lr * nabla for w, nabla in zip(self.weights, nabla_w)]
self.biases = [b - lr * nabla for b, nabla in zip(self.biases, nabla_b)]
return loss
def evaluate(self, test_data):
'''
:param test_data: list of (x, y)
:return:
'''
result = [(np.argmax(self.forward(x)), y) for x, y in test_data]
correct = sum(int(pred==y) for pred, y in result)
return correct
def main():
import mnist_loader
# loading MNIST data
training_data, validation_data, test_data = mnist_loader.load_data_wrapper()
print(len(training_data), training_data[0][0].shape, training_data[0][1].shape)
print(len(test_data), test_data[0][0].shape, test_data[0][1].shape)
# Set up a network with 30 hidden neurons
net = MLP_np([784, 30, 10])
# Use SGD to learn from the MNIST training data
# 1000 epochs, with a mini-batch size of 10, lr=0.1
net.train(training_data, 1000, 10, 0.1, test_data=test_data)
if __name__ == '__main__':
main()
>>> Epoch 999: 9436 / 10000 0.0002412281437943844