参考大神的文章https://www.zybuluo.com/hanbingtao/note/476663
感觉逐渐入门了深度学习,在这写点现阶段自己关于对神经网络和反向传播算法的理解。简单的神经网络通常有三层:输入层,隐藏层,输出层。如果不考虑复杂的数学公式推导的话,其实没有多少东西,只要弄清每层的输入输出,每层的连接方式,其实并没有多困难,输入输出比较容易弄清楚,输入层的输出就是隐藏层的输入,以此推,很容易理解。连接方式,无非是每一层是全连接还是连接一部分,连接的时候每一个神经元的权重是多少的问题。最难的部分在于根据一定的误差准则去确定权重,这个时候需要复杂的数学公式进行推导,一般是链式求导。在求导的时候会根据一定的算法,比如反向传播算法等,求导完成之后又会进一步去优化,优化的算法又会有很多种,有随机梯度下降算法等(自己还没掌握那么多算法,暂且写自己掌握的)最后得到我们求更新权重的公式,然后根据公式,用python或者c++等等其他任何一种语言去敲出来,大功告成。贴上大神的代码基于python2.7。方便以后进一步学习。
import random
import numpy as np
from activators import SigmoidActivator, IdentityActivator
# 全连接层实现类
class FullConnectedLayer(object):
def __init__(self, input_size, output_size,
activator):
'''
构造函数
input_size: 本层输入向量的维度
output_size: 本层输出向量的维度
activator: 激活函数
'''
self.input_size = input_size
self.output_size = output_size
self.activator = activator
# 权重数组W
self.W = np.random.uniform(-0.1, 0.1,
(output_size, input_size))
# 偏置项b
self.b = np.zeros((output_size, 1))
# 输出向量
self.output = np.zeros((output_size, 1))
def forward(self, input_array):
'''
前向计算
input_array: 输入向量,维度必须等于input_size
'''
# 式2
self.input = input_array
self.output = self.activator.forward(
np.dot(self.W, input_array) + self.b)
def backward(self, delta_array):
'''
反向计算W和b的梯度
delta_array: 从上一层传递过来的误差项
'''
# 式8
self.delta = self.activator.backward(self.input) * np.dot(
self.W.T, delta_array)
self.W_grad = np.dot(delta_array, self.input.T)
self.b_grad = delta_array
def update(self, learning_rate):
'''
使用梯度下降算法更新权重
'''
self.W += learning_rate * self.W_grad
self.b += learning_rate * self.b_grad
def dump(self):
print 'W: %s\nb:%s' % (self.W, self.b)
# 神经网络类
class Network(object):
def __init__(self, layers):
'''
构造函数
'''
self.layers = []
for i in range(len(layers) - 1):
self.layers.append(
FullConnectedLayer(
layers[i], layers[i+1],
SigmoidActivator()
)
)
def predict(self, sample):
'''
使用神经网络实现预测
sample: 输入样本
'''
output = sample
for layer in self.layers:
layer.forward(output)
output = layer.output
return output
def train(self, labels, data_set, rate, epoch):
'''
训练函数
labels: 样本标签
data_set: 输入样本
rate: 学习速率
epoch: 训练轮数
'''
for i in range(epoch):
for d in range(len(data_set)):
self.train_one_sample(labels[d],
data_set[d], rate)
def train_one_sample(self, label, sample, rate):
self.predict(sample)
self.calc_gradient(label)
self.update_weight(rate)
def calc_gradient(self, label):
delta = self.layers[-1].activator.backward(
self.layers[-1].output
) * (label - self.layers[-1].output)
for layer in self.layers[::-1]:
layer.backward(delta)
delta = layer.delta
return delta
def update_weight(self, rate):
for layer in self.layers:
layer.update(rate)
def dump(self):
for layer in self.layers:
layer.dump()
def loss(self, output, label):
return 0.5 * ((label - output) * (label - output)).sum()
def gradient_check(self, sample_feature, sample_label):
'''
梯度检查
network: 神经网络对象
sample_feature: 样本的特征
sample_label: 样本的标签
'''
# 获取网络在当前样本下每个连接的梯度
self.predict(sample_feature)
self.calc_gradient(sample_label)
# 检查梯度
epsilon = 10e-4
for fc in self.layers:
for i in range(fc.W.shape[0]):
for j in range(fc.W.shape[1]):
fc.W[i,j] += epsilon
output = self.predict(sample_feature)
err1 = self.loss(sample_label, output)
fc.W[i,j] -= 2*epsilon
output = self.predict(sample_feature)
err2 = self.loss(sample_label, output)
expect_grad = (err1 - err2) / (2 * epsilon)
fc.W[i,j] += epsilon
print 'weights(%d,%d): expected - actural %.4e - %.4e' % (
i, j, expect_grad, fc.W_grad[i,j])
from bp import train_data_set
def transpose(args):
return map(
lambda arg: map(
lambda line: np.array(line).reshape(len(line), 1)
, arg)
, args
)
class Normalizer(object):
def __init__(self):
self.mask = [
0x1, 0x2, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80
]
def norm(self, number):
data = map(lambda m: 0.9 if number & m else 0.1, self.mask)
return np.array(data).reshape(8, 1)
def denorm(self, vec):
binary = map(lambda i: 1 if i > 0.5 else 0, vec[:,0])
for i in range(len(self.mask)):
binary[i] = binary[i] * self.mask[i]
return reduce(lambda x,y: x + y, binary)
def train_data_set():
normalizer = Normalizer()
data_set = []
labels = []
for i in range(0, 256):
n = normalizer.norm(i)
data_set.append(n)
labels.append(n)
return labels, data_set
def correct_ratio(network):
normalizer = Normalizer()
correct = 0.0;
for i in range(256):
if normalizer.denorm(network.predict(normalizer.norm(i))) == i:
correct += 1.0
print 'correct_ratio: %.2f%%' % (correct / 256 * 100)
def test():
labels, data_set = transpose(train_data_set())
net = Network([8, 3, 8])
rate = 0.5
mini_batch = 20
epoch = 10
for i in range(epoch):
net.train(labels, data_set, rate, mini_batch)
print 'after epoch %d loss: %f' % (
(i + 1),
net.loss(labels[-1], net.predict(data_set[-1]))
)
rate /= 2
correct_ratio(net)
def gradient_check():
'''
梯度检查
'''
labels, data_set = transpose(train_data_set())
net = Network([8, 3, 8])
net.gradient_check(data_set[0], labels[0])
return net