资料:
http://blog.csdn.net/stdcoutzyx/article/details/41596663
http://blog.csdn.net/zouxy09/article/details/8775488
BP算法:
http://blog.csdn.net/marvin521/article/details/9886643
numpy的float64的精度比不上python自带的float类型,所以结果比不上原文中的实现。
# coding: utf-8
import numpy as np
import math
import random
np.random.seed(0)
random.seed(0)
# calculate a random number where: a <= rand < b
def rand(a, b):
return (b-a)*random.random() + a
def sigmoid(x):
# f(z) = (e ^ z - e ^ (-z)) / (e ^ z + e ^ (-z))
# http://ufldl.stanford.edu/wiki/index.php/%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C
return math.tanh(x)
def dsigmoid(y):
# 导数为:f'(z) = 1 - f(z) ^ 2
return 1.0 - y ** 2
class NN(object):
def __init__(self, ni, nh, no):
# 结点数
self.ni = ni + 1
self.nh = nh
self.no = no
# 值
self.ai = np.ones((self.ni,))
self.ah = np.ones((self.nh,))
self.ao = np.ones((self.no,))
# 权重
# self.wi = np.random.uniform(-0.2, 0.2, self.ni * self.nh).reshape(self.ni, self.nh)
# self.wo = np.random.uniform(2., -2., self.nh * self.no).reshape(self.nh, self.no)
self.wi = np.zeros((self.ni, self.nh))
self.wo = np.zeros((self.nh, self.no))
for i in range(self.ni):
for j in range(self.nh):
self.wi[i][j] = rand(-0.2, 0.2)
# print(self.wi[i][j].dtype)
for i in range(self.nh):
for j in range(self.no):
self.wo[i][j] = rand(-2.0, 2.0)
# 旧的weight
self.ci = np.zeros((self.ni, self.nh))
self.co = np.zeros((self.nh, self.no))
def update(self, inputs):
assert(len(inputs) == self.ni - 1)
for i in range(self.ni - 1):
self.ai[i] = inputs[i]
for i in range(self.nh):
s = 0
for j in range(self.ni):
s += self.ai[j] * self.wi[j][i]
self.ah[i] = sigmoid(s)
for i in range(self.no):
s = 0
for j in range(self.nh):
s += self.ah[j] * self.wo[j][i]
self.ao[i] = sigmoid(s)
def back_propagate(self, targets, N, M):
assert(len(targets) == self.no)
# y = sigmoid(a2 + b), J = 0.5 * (y - t) ** 2, delta_J = (y - t) * y ' * h
# output_delta = (y - t) * y'
output_deltas = np.zeros(self.no)
for i in range(self.no):
err = targets[i] - self.ao[i]
output_deltas[i] = dsigmoid(self.ao[i]) * err
# 这里取两次的delta来逐步改变
# delta_j = (y - t) * y' * ah
# W_new = W_old + r1 * delta_J + r2 * delta_J_old
for i in range(self.nh):
for j in range(self.no):
change = output_deltas[j] * self.ah[i]
self.wo[i][j] += N * change + M * self.co[i][j]
self.co[i][j] = change
# hidden_delta = (y - t) * y' * Wo * h'
# delta_J = (y - t) * y' * Wo * h' * ai
hidden_deltas = np.zeros((self.nh,))
for i in range(self.nh):
err = 0
for j in range(self.no):
err += output_deltas[j] * self.wo[i][j]
hidden_deltas[i] = dsigmoid(self.ah[i]) * err
# 更新input weight
for i in range(self.ni):
for j in range(self.nh):
change = hidden_deltas[j] * self.ai[i]
self.wi[i][j] += N * change + M * self.ci[i][j]
self.ci[i][j] = change
# 计算错误率
err = 0.
for i in range(len(targets)):
err += 0.5 * (targets[i] - self.ao[i]) ** 2
return err
def train(self, patterns, iterations=1000, N=0.5, M=0.1):
for i in range(iterations):
err = 0.
for p in patterns:
inputs = p[0]
targets = p[1]
self.update(inputs)
err += self.back_propagate(targets, N, M)
if i % 100 == 0:
self.weights()
print('error %-.5f' % err)
def weights(self):
print("\nInput weights:")
for i in range(self.ni):
print(self.wi[i])
print("\nOutput weights")
for i in range(self.nh):
print(self.wo[i])
def main():
pat = np.array([
[[0,0], [0]],
[[0,1], [1]],
[[1,0], [1]],
[[1,1], [0]]
])
nn = NN(2, 2, 1)
nn.train(pat)
if __name__ == "__main__":
main()
注意:
python dict的*操作只是浅拷贝:
t = [0] * 3
h = [t] * 5
h[0][0] = 1
print(h)
# [[1, 0, 0], [1, 0, 0], [1, 0, 0], [1, 0, 0], [1, 0, 0]]
[[0]],dict里面的对象只是引用,并没有copy。
检查了好多遍之后,发现上面不是由于误差引起的。而是计算delta的顺序错了,梯度下降是需要协调改变的,所以需要先算完两个delta值,再更新W。
# coding: utf-8
import numpy as np
import math
import random
np.random.seed(0)
# random.seed(0)
# calculate a random number where: a <= rand < b
def rand(a, b):
return (b-a)*random.random() + a
# Make a matrix (we could use NumPy to speed this up)
def makeMatrix(I, J, fill=0.0):
m = []
for i in range(I):
m.append([fill]*J)
return m
def sigmoid(x):
# f(z) = (e ^ z - e ^ (-z)) / (e ^ z + e ^ (-z))
# http://ufldl.stanford.edu/wiki/index.php/%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C
return math.tanh(x)
def dsigmoid(y):
# 导数为:f'(z) = 1 - f(z) ^ 2
return 1.0 - y ** 2
class NN(object):
def __init__(self, ni, nh, no):
# 结点数
self.ni = ni + 1
self.nh = nh
self.no = no
# 值
self.ai = np.ones((self.ni,))
self.ah = np.ones((self.nh,))
self.ao = np.ones((self.no,))
# self.ai = [1.] * self.ni
# self.ah = [1.] * self.nh
# self.ao = [1.] * self.no
# 权重
self.wi = np.random.uniform(-0.2, 0.2, self.ni * self.nh).reshape(self.ni, self.nh)
self.wo = np.random.uniform(2., -2., self.nh * self.no).reshape(self.nh, self.no)
# self.wi = np.zeros((self.ni, self.nh))
# self.wo = np.zeros((self.nh, self.no))
# self.wi = makeMatrix(self.ni, self.nh)
# self.wo = makeMatrix(self.nh, self.no)
#
# for i in range(self.ni):
# for j in range(self.nh):
# self.wi[i][j] = rand(-0.2, 0.2)
# # print(type(self.wi[i][j]))
#
# for i in range(self.nh):
# for j in range(self.no):
# self.wo[i][j] = rand(-2.0, 2.0)
# 旧的weight
self.ci = np.zeros((self.ni, self.nh))
self.co = np.zeros((self.nh, self.no))
# self.ci = makeMatrix(self.ni, self.nh)
# self.co = makeMatrix(self.nh, self.no)
def update(self, inputs):
assert(len(inputs) == self.ni - 1)
for i in range(self.ni - 1):
self.ai[i] = inputs[i]
for i in range(self.nh):
s = 0.
for j in range(self.ni):
s += self.ai[j] * self.wi[j][i]
self.ah[i] = sigmoid(s)
for i in range(self.no):
s = 0.
for j in range(self.nh):
s += self.ah[j] * self.wo[j][i]
self.ao[i] = sigmoid(s)
def back_propagate(self, targets, N, M):
assert(len(targets) == self.no)
# y = sigmoid(a2 + b), J = 0.5 * (y - t) ** 2, delta_J = (y - t) * y ' * h
# output_delta = (y - t) * y'
output_deltas = np.zeros(self.no)
# output_deltas = [0.] * self.no
# print(output_deltas)
for i in range(self.no):
err = targets[i] - self.ao[i]
output_deltas[i] = dsigmoid(self.ao[i]) * err
# hidden_delta = (y - t) * y' * Wo * h'
# delta_J = (y - t) * y' * Wo * h' * ai
hidden_deltas = np.zeros(self.nh)
# hidden_deltas = [0.] * self.nh
# print(hidden_deltas)
for i in range(self.nh):
err = 0.
for j in range(self.no):
err += output_deltas[j] * self.wo[i][j]
hidden_deltas[i] = dsigmoid(self.ah[i]) * err
# 这里取两次的delta来逐步改变
# delta_j = (y - t) * y' * ah
# W_new = W_old + r1 * delta_J + r2 * delta_J_old
for i in range(self.nh):
for j in range(self.no):
change = output_deltas[j] * self.ah[i]
self.wo[i][j] += N * change + M * self.co[i][j]
self.co[i][j] = change
# 更新input weight
for i in range(self.ni):
for j in range(self.nh):
change = hidden_deltas[j] * self.ai[i]
self.wi[i][j] += N * change + M * self.ci[i][j]
self.ci[i][j] = change
# 计算错误率
err = 0.
for i in range(len(targets)):
err += 0.5 * (targets[i] - self.ao[i]) ** 2
return err
def train(self, patterns, iterations=1000, N=0.5, M=0.1):
for i in range(iterations):
err = 0.
for p in patterns:
inputs = p[0]
targets = p[1]
self.update(inputs)
err += self.back_propagate(targets, N, M)
if i % 100 == 0:
# self.weights()
print('error %-.5f' % err)
def weights(self):
print("\nInput weights:")
for i in range(self.ni):
print(self.wi[i])
print("\nOutput weights:")
for i in range(self.nh):
print(self.wo[i])
def main():
pat = np.array([
[[0,0], [0]],
[[0,1], [1]],
[[1,0], [1]],
[[1,1], [0]]
])
nn = NN(2, 2, 1)
# nn.weights()
nn.train(pat)
if __name__ == "__main__":
main()
numpy的float64精确度还是不够,最后结果有0.1的误差