反向传播(Back Propagation,简称BP)神经网络解决了多层神经网络的学习问题,广泛应用于分类识别、图像识别、压缩、逼近以及回归等领域,其结构如下所示。
另外介绍及格激活函数:sigmoid、tanh和softsign。神经网络中的激活函数,其作用就是引入非线性。
Sigmoid:sigmoid的优点是输出范围有限,数据在传递的过程中不容易发散,求导很容易(y=sigmoid(x), y’=y(1-y))。缺点是饱和的时候梯度太小。其输出范围为(0, 1),所以可以用作输出层,输出表示概率。
公式:
tanh和softsign:
使用BP神经网络解决异或问题:
使用sigmoid激活函数,偏置值定为1。
import numpy as np
import matplotlib.pyplot as plt
#输入数据
X = np.array([[1,0,0],
[1,0,1],
[1,1,0],
[1,1,1]])
#标记
Y = np.array([[0,1,1,0]])
#权值初始化,取值范围为-1到1
V = np.random.random((3,4))*2-1#第一层权值
W = np.random.random((4,1))*2-1#第二层权值
print(V)
print(W)
#学习率
lr = 0.11
#sigmoid激活函数
def sigmoid(x):
return 1/(1+np.exp(-x))
#sigmoid函数导数
def dsigmoid(x):
return x*(1-x)
#更新权值
def update():
global X,V,W,lr
L1 = sigmoid(np.dot(X,V))#隐藏层输出--4x4的矩阵
L2 = sigmoid(np.dot(L1,W))#输出层输出--4x1的矩阵
L2_delta = (Y.T - L2)*dsigmoid(L2)
L1_delta = np.dot(L2_delta,W.T)*dsigmoid(L1)
W_C = lr*np.dot(L1.T,L2_delta)
V_C = lr*np.dot(X.T,L1_delta)
W = W + W_C
V = V + V_C
[[-0.63777659 -0.54506959 -0.33179877 -0.31877276]
[-0.95125978 0.7730579 0.29587091 -0.58236583]
[ 0.79077282 -0.93217298 -0.09382096 0.5822262 ]]
[[ 0.59146382]
[-0.30789231]
[-0.57723785]
[ 0.18645706]]
for i in range(10000):#迭代一万次
update()#更新权值
if i%500 == 0:#每500次打印一下误差
L1 = sigmoid(np.dot(X,V))#隐藏层输出--4x4的矩阵
L2 = sigmoid(np.dot(L1,W))#输出层输出--4x1的矩阵
print("Error:",np.mean(np.abs(Y.T-L2)))
L1 = sigmoid(np.dot(X,V))#隐藏层输出--4x4的矩阵
L2 = sigmoid(np.dot(L1,W))#输出层输出--4x1的矩阵
print(L2)
Error: 0.498712507036
Error: 0.494796348535
Error: 0.480546625904
Error: 0.4198798183
Error: 0.279981268445
Error: 0.178200802236
Error: 0.130067381817
Error: 0.10418018522
Error: 0.0881430063965
Error: 0.0771905626373
Error: 0.0691913263575
Error: 0.0630610081972
Error: 0.0581913640284
Error: 0.0542146972308
Error: 0.0508953510674
Error: 0.0480750732027
Error: 0.0456434672314
Error: 0.0435210496818
Error: 0.0416490513475
Error: 0.0399830217768
[[ 0.04218605]
[ 0.95982694]
[ 0.96413189]
[ 0.03573883]]
最后的输出结果也很接近[0,1,1,0]
使用BP神经网络解决数字显示问题:
from sklearn.datasets import load_digits
import numpy as np
from sklearn.preprocessing import LabelBinarizer
from sklearn.cross_validation import train_test_split
def sigmoid(x):
return 1/(1+np.exp(-x))
def dsigmoid(x):
return x*(1-x)
class NeuralNetwort:
def __init__(self,layers):#(640,100,10)
#权值的初始化,范围-1到1.
self.V = np.random.random((layers[0]+1,layers[1]+1))*2-1
self.W = np.random.random((layers[1]+1,layers[2]))*2-1
def train(self,X,y,lr=0.11,epochs=10000):
#添加偏置
temp = np.ones([X.shape[0],X.shape[1]+1])
temp[:,0:-1] = X
X = temp
for n in range(epochs+1):
i = np.random.randint(X.shape[0])#随机选取一个数据
x = [X[i]]
x = np.atleast_2d(x)#转为2维数据
L1 = sigmoid(np.dot(x,self.V))#隐藏层输出
L2 = sigmoid(np.dot(L1,self.W))#输出层输出
L2_delta = (y[i]-L2)*dsigmoid(L2)
L1_delta = np.dot(L2_delta,self.W.T)*dsigmoid(L1)
self.W += lr*np.dot(L1.T,L2_delta)
self.V += lr*np.dot(x.T,L1_delta)
#每训练一千次预测一次准确率
if n%1000 == 0:
predictions = []
for j in range(X_test.shape[0]):
o=self.predict(X_test[j])
predictions.append(np.argmax(o))#获取预测结果
accuracy = np.mean(np.equal(predictions,y_test))
print("epoch:",n,"accuracy:",accuracy)
def predict(self,x):
#添加偏置
temp = np.ones(x.shape[0]+1)
temp[0:-1] = x
x = temp
x = np.atleast_2d(x)#转为2维数据
L1 = sigmoid(np.dot(x,self.V))#隐藏层输出
L2 = sigmoid(np.dot(L1,self.W))#输出层输出
return L2
digits = load_digits()#载入数据
X = digits.data#数据
y = digits.target#标记
#输入数据归一化
X -= X.min()
X /= X.max()
nm = NeuralNetwort([64,100,10])#创建网络
X_train,X_test,y_train,y_test = train_test_split(X,y)#分割1/4数据为测试数据,3/4为训练数据。
labels_train = LabelBinarizer().fit_transform(y_train)#标记二值化
labels_test = LabelBinarizer().fit_transform(y_test)#标记二值化
print("start")
nm.train(X_train,labels_train,epochs=20000)
print("end")
start
epoch: 0 accuracy: 0.16
epoch: 1000 accuracy: 0.688888888889
epoch: 2000 accuracy: 0.895555555556
epoch: 3000 accuracy: 0.897777777778
epoch: 4000 accuracy: 0.904444444444
epoch: 5000 accuracy: 0.953333333333
epoch: 6000 accuracy: 0.931111111111
epoch: 7000 accuracy: 0.946666666667
epoch: 8000 accuracy: 0.946666666667
epoch: 9000 accuracy: 0.962222222222
epoch: 10000 accuracy: 0.964444444444
epoch: 11000 accuracy: 0.957777777778
epoch: 12000 accuracy: 0.96
epoch: 13000 accuracy: 0.955555555556
epoch: 14000 accuracy: 0.962222222222
epoch: 15000 accuracy: 0.968888888889
epoch: 16000 accuracy: 0.964444444444
epoch: 17000 accuracy: 0.948888888889
epoch: 18000 accuracy: 0.955555555556
epoch: 19000 accuracy: 0.957777777778
epoch: 20000 accuracy: 0.966666666667
end