BP神经网络

反向传播（Back Propagation，简称BP）神经网络解决了多层神经网络的学习问题，广泛应用于分类识别、图像识别、压缩、逼近以及回归等领域，其结构如下所示。

另外介绍及格激活函数：sigmoid、tanh和softsign。神经网络中的激活函数，其作用就是引入非线性。
Sigmoid：sigmoid的优点是输出范围有限，数据在传递的过程中不容易发散，求导很容易（y=sigmoid(x), y’=y(1-y)）。缺点是饱和的时候梯度太小。其输出范围为(0, 1)，所以可以用作输出层，输出表示概率。

公式：

tanh和softsign：

使用BP神经网络解决异或问题：
使用sigmoid激活函数，偏置值定为1。

import numpy as np
import matplotlib.pyplot as plt

#输入数据
X = np.array([[1,0,0],
              [1,0,1],
              [1,1,0],
              [1,1,1]])
#标记 
Y = np.array([[0,1,1,0]])
#权值初始化，取值范围为-1到1
V = np.random.random((3,4))*2-1#第一层权值
W = np.random.random((4,1))*2-1#第二层权值
print(V)
print(W)
#学习率
lr = 0.11

#sigmoid激活函数
def sigmoid(x):
    return 1/(1+np.exp(-x))
#sigmoid函数导数
def dsigmoid(x):
    return x*(1-x)
#更新权值
def update():
    global X,V,W,lr

    L1 = sigmoid(np.dot(X,V))#隐藏层输出--4x4的矩阵
    L2 = sigmoid(np.dot(L1,W))#输出层输出--4x1的矩阵

    L2_delta = (Y.T - L2)*dsigmoid(L2)
    L1_delta = np.dot(L2_delta,W.T)*dsigmoid(L1)

    W_C = lr*np.dot(L1.T,L2_delta)
    V_C = lr*np.dot(X.T,L1_delta)

    W = W + W_C
    V = V + V_C

[[-0.63777659 -0.54506959 -0.33179877 -0.31877276]
 [-0.95125978  0.7730579   0.29587091 -0.58236583]
 [ 0.79077282 -0.93217298 -0.09382096  0.5822262 ]]
[[ 0.59146382]
 [-0.30789231]
 [-0.57723785]
 [ 0.18645706]]

for i in range(10000):#迭代一万次
    update()#更新权值
    if i%500 == 0:#每500次打印一下误差
        L1 = sigmoid(np.dot(X,V))#隐藏层输出--4x4的矩阵
        L2 = sigmoid(np.dot(L1,W))#输出层输出--4x1的矩阵
        print("Error:",np.mean(np.abs(Y.T-L2)))

L1 = sigmoid(np.dot(X,V))#隐藏层输出--4x4的矩阵
L2 = sigmoid(np.dot(L1,W))#输出层输出--4x1的矩阵
print(L2)

Error: 0.498712507036
Error: 0.494796348535
Error: 0.480546625904
Error: 0.4198798183
Error: 0.279981268445
Error: 0.178200802236
Error: 0.130067381817
Error: 0.10418018522
Error: 0.0881430063965
Error: 0.0771905626373
Error: 0.0691913263575
Error: 0.0630610081972
Error: 0.0581913640284
Error: 0.0542146972308
Error: 0.0508953510674
Error: 0.0480750732027
Error: 0.0456434672314
Error: 0.0435210496818
Error: 0.0416490513475
Error: 0.0399830217768
[[ 0.04218605]
 [ 0.95982694]
 [ 0.96413189]
 [ 0.03573883]]

最后的输出结果也很接近[0,1,1,0]

---

使用BP神经网络解决数字显示问题：

from sklearn.datasets import load_digits
import numpy as np
from sklearn.preprocessing import LabelBinarizer
from sklearn.cross_validation import train_test_split

def sigmoid(x):
    return 1/(1+np.exp(-x))

def dsigmoid(x):
    return x*(1-x)

class NeuralNetwort:
    def __init__(self,layers):#(640,100,10)
        #权值的初始化，范围-1到1.
        self.V = np.random.random((layers[0]+1,layers[1]+1))*2-1
        self.W = np.random.random((layers[1]+1,layers[2]))*2-1        

    def train(self,X,y,lr=0.11,epochs=10000):
        #添加偏置
        temp = np.ones([X.shape[0],X.shape[1]+1])
        temp[:,0:-1] = X
        X = temp

        for n in range(epochs+1):
            i = np.random.randint(X.shape[0])#随机选取一个数据
            x = [X[i]]
            x = np.atleast_2d(x)#转为2维数据

            L1 = sigmoid(np.dot(x,self.V))#隐藏层输出
            L2 = sigmoid(np.dot(L1,self.W))#输出层输出

            L2_delta = (y[i]-L2)*dsigmoid(L2)
            L1_delta = np.dot(L2_delta,self.W.T)*dsigmoid(L1)

            self.W += lr*np.dot(L1.T,L2_delta)
            self.V += lr*np.dot(x.T,L1_delta)

            #每训练一千次预测一次准确率
            if n%1000 == 0:
                predictions = []
                for j in range(X_test.shape[0]):
                    o=self.predict(X_test[j])
                    predictions.append(np.argmax(o))#获取预测结果
                accuracy = np.mean(np.equal(predictions,y_test))
                print("epoch:",n,"accuracy:",accuracy)
    def predict(self,x):
    #添加偏置
        temp = np.ones(x.shape[0]+1)
        temp[0:-1] = x
        x = temp
        x = np.atleast_2d(x)#转为2维数据

        L1 = sigmoid(np.dot(x,self.V))#隐藏层输出
        L2 = sigmoid(np.dot(L1,self.W))#输出层输出
        return L2

digits = load_digits()#载入数据
X = digits.data#数据
y = digits.target#标记

#输入数据归一化
X -= X.min()
X /= X.max()

nm = NeuralNetwort([64,100,10])#创建网络
X_train,X_test,y_train,y_test = train_test_split(X,y)#分割1/4数据为测试数据，3/4为训练数据。
labels_train = LabelBinarizer().fit_transform(y_train)#标记二值化
labels_test = LabelBinarizer().fit_transform(y_test)#标记二值化      
print("start")
nm.train(X_train,labels_train,epochs=20000)
print("end")

start
epoch: 0 accuracy: 0.16
epoch: 1000 accuracy: 0.688888888889
epoch: 2000 accuracy: 0.895555555556
epoch: 3000 accuracy: 0.897777777778
epoch: 4000 accuracy: 0.904444444444
epoch: 5000 accuracy: 0.953333333333
epoch: 6000 accuracy: 0.931111111111
epoch: 7000 accuracy: 0.946666666667
epoch: 8000 accuracy: 0.946666666667
epoch: 9000 accuracy: 0.962222222222
epoch: 10000 accuracy: 0.964444444444
epoch: 11000 accuracy: 0.957777777778
epoch: 12000 accuracy: 0.96
epoch: 13000 accuracy: 0.955555555556
epoch: 14000 accuracy: 0.962222222222
epoch: 15000 accuracy: 0.968888888889
epoch: 16000 accuracy: 0.964444444444
epoch: 17000 accuracy: 0.948888888889
epoch: 18000 accuracy: 0.955555555556
epoch: 19000 accuracy: 0.957777777778
epoch: 20000 accuracy: 0.966666666667
end