BP神经网络推导

示意图

符号说明

$$\begin{array}{rl}{\mathbf{y}}^0:& 输入，\mathbf{y}\in {\mathbb{R}}^{s0\times 1}\\ {\mathbf{z}}^l:& 第l层输出{\mathbf{z}}^{(l)}\in {\mathbb{R}}^{sl\times 1}\\ {\mathbf{y}}^l:& 第l层输出{\mathbf{y}}^{(l)}\in {\mathbb{R}}^{sl\times 1}\\ \mathbf{\sigma }:& 激活函数\\ sl:& 表示l层{\mathbf{y}}^{(l)}{\mathbf{z}}^{(l)}的向量维数\\ \mathbf{t}:& 表示真实值\\ L:& 一共L层\\ f_i^l:& 表示\frac{\partial y_i^l}{\partial z_i^l}\\ \mathbf{I}(i):& 表示为列向量，且在第i行为1，其余位置为0;\\ {\delta }_i^l:& 表示\frac{\partial E}{\partial y_i^l}\\ {\mathbf{\delta }}^l:& 表示\frac{\partial E}{\partial {\mathbf{y}}^l}，即为：\left(\begin{array}{c}{\delta }_1^l,{\delta }_2^l,\cdots ,{\delta }_{sl}^l\end{array}\right)\end{array}$$
它们之间的关系：
$$\begin{array}{rl}{\mathbf{z}}^l& ={\mathbf{w}}^l\ast {\mathbf{y}}^{l-1}\\ z_i^l& =\sum _{j=1}^{s(l-1)}{w}_{ij}\ast y_j^{l-1}\\ {\mathbf{y}}^l& =\mathbf{\sigma }({\mathbf{y}}^l)\\ y_i^l& =\sigma (z_i^l)\end{array}f(\mathbf{x})$$

矩阵相关求导说明

符号说明
$\mathbf{y}:列向量,\mathbf{y}\in {\mathbb{R}}^{n\times 1}$
$\mathbf{x}:列向量,\mathbf{x}\in {\mathbb{R}}^{m\times 1}$
$f(\mathbf{x}):实值标量函数，记做f:{\mathbb{R}}^m\to \mathbb{R}$
公式
$$\begin{array}{rl}\frac{\partial {\mathbf{y}}^{\mathrm{T}}}{\partial \mathbf{x}}& =\left(\begin{array}{ccc}\frac{\partial y_1}{\partial x_1}& \cdots & \frac{\partial y_n}{\partial x_1}\\ ⋮& & ⋮\\ \frac{\partial y_1}{\partial x_m}& \cdots & \frac{\partial y_n}{\partial x_m}\end{array}\right)\\ \frac{\partial {\mathbf{y}}^{\mathrm{T}}}{\partial \mathbf{y}}& ={\mathbf{E}}_{n\times n}\\ \frac{f(\mathbf{x})}{\partial \mathbf{x}}& =[\frac{f(\mathbf{x})}{\partial x_1},\cdots ,\frac{f(\mathbf{x})}{\partial x_m}{]}^{\mathrm{T}}\end{array}$$

公式推导

误差定义
$$\begin{array}{rl}E& =\frac{1}{m}\sum _{p=1}^m(E_p)\\ E_p& =\frac{1}{2}({\mathbf{y}}^L-{\mathbf{t}}^L{)}^2\\ & =\frac{1}{2}\sum _{i=1}^{sL}(y_i^L-t_i{)}^2\end{array}$$

其中m为样本数，为了推导简单，让m=1
求$\frac{\partial E}{\partial w_{ij}^L}$
几点说明
$$\begin{array}{rl}\frac{\partial z_k^l}{\partial w_{ij}^l}& =\{\begin{array}{ll}{z}_j^{l-1}& z=i\\ 0& k̸=i\end{array}\\ \frac{\partial {\mathbf{z}}^l}{\partial w_{ij}}& =[\frac{\partial z_1^l}{\partial w_{ij}^l},\cdots ,\frac{\partial z_{sl}^l}{\partial w_{ij}^l}{]}^{\mathrm{T}}\in {\mathbb{R}}^{sl\times 1}\\ & =\mathbf{I}(i).z_i^{l-1}\\ & \\ \frac{\partial {\mathbf{y}}^l}{\partial ({\mathbf{z}}^l{)}^{\mathrm{T}}}& =\left(\begin{array}{ccc}\frac{\partial y_1^l}{\partial z_1^l}& \cdots & \frac{\partial y_1^l}{\partial z_{sl}^l}\\ ⋮& & ⋮\\ \frac{\partial y_{sl}^l}{\partial z_1^l}& \cdots & \frac{\partial y_{sl}^l}{\partial z_{sl}^l}\end{array}\right)\in {\mathbb{R}}^{sl\times sl}\\ & =\left(\begin{array}{cccc}{f}_1^l& & & \\ & f_2^l& & \\ & & \ddots & \\ & & & f_{(sl)}^l\end{array}\right)\\ & \\ \frac{\partial {\mathbf{z}}^l}{\partial ({\mathbf{y}}^{l-1}{)}^{\mathrm{T}}}& =\left(\begin{array}{ccc}\frac{\partial z_1^l}{\partial y_1^{l-1}}& \cdots & \frac{\partial z_1^l}{\partial y_{s(l-1)}^{l-1}}\\ ⋮& & ⋮\\ \frac{\partial z_{sl}^l}{\partial y_1^{l-1}}& \cdots & \frac{\partial z_{sl}^l}{\partial y_{s(l-1)}^{(l-1)}}\end{array}\right)\\ & =\left(\begin{array}{ccc}{w}_{11}^l& \cdots & w_{(s(l-1))1}^l\\ ⋮& & ⋮\\ w_{(sl)1}^l& \cdots & w_{(sl)(s(l-1))}^l\end{array}\right)\in {\mathbb{R}}^{sl\times s(l-1)}\\ & \\ \frac{\partial {\mathbf{y}}^l}{\partial {\mathbf{y}}^{l-1}}& =\frac{\partial {\mathbf{y}}^l}{\partial {\mathbf{z}}^l}.\frac{\partial {\mathbf{z}}^l}{\partial {\mathbf{y}}^{l-1}}\\ & =\frac{\partial {\mathbf{y}}^l}{\partial ({\mathbf{z}}^l{)}^{\mathrm{T}}}.\frac{\partial ({\mathbf{z}}^l{)}^{\mathrm{T}}}{\partial {\mathbf{z}}^l}.\frac{\partial {\mathbf{z}}^l}{\partial ({\mathbf{y}}^{l-1}{)}^{\mathrm{T}}}.\frac{\partial ({\mathbf{y}}^{l-1}{)}^{\mathrm{T}}}{\partial {\mathbf{y}}^{l-1}}\\ & =\frac{\partial {\mathbf{y}}^l}{\partial ({\mathbf{z}}^l{)}^{\mathrm{T}}}.\frac{\partial {\mathbf{z}}^l}{\partial ({\mathbf{y}}^{l-1}{)}^{\mathrm{T}}}\\ & =\left(\begin{array}{ccc}{f}_1^l{w}_{11}^l& \cdots & f_1^l{w}_{(s(l-1))1}\\ ⋮& & ⋮\\ f_{sl}^l{w}_{(sl)1}^l& \cdots & f_{sl}^l{w}_{(sl)(s(l-1))}^l\end{array}\right)\in {\mathbb{R}}^{sl\times s(l-1)}\\ & \\ \frac{\partial E}{\partial ({\mathbf{y}}^L{)}^{\mathrm{T}}}& =[y_1^L-t_1,\cdots ,y_{sL}^L-t_{sL}]\\ & \\ \frac{\partial {\mathbf{z}}^l}{\partial z_i^{l-1}}& =[w_{1i}^l,w_{2i}^l,\cdots ,w_{(sl)i}^l{]}^{\mathrm{T}}\\ & \\ \frac{\partial {\mathbf{y}}^l}{\partial y_i^{l-1}}& =\frac{\partial {\mathbf{y}}^l}{\partial {\mathbf{z}}^l}.\frac{\partial {\mathbf{z}}^l}{\partial y_i^{l-1}}\\ & =\left(\begin{array}{cccc}{f}_1^l& & & \\ & f_2^l& & \\ & & \ddots & \\ & & & f_{(sl)}^l\end{array}\right).\left(\begin{array}{c}{w}_{1i}^l\\ w_{2i}^l\\ \cdots \\ w_{(sl)i}^l\end{array}\right)\\ & =\left(\begin{array}{c}{f}_1^l{w}_{1i}^l\\ f_2^l{w}_{2i}^l\\ ⋮\\ f_{(sl)}^l{w}_{(sl)i}^l\end{array}\right)\end{array}$$

求解
$$\begin{array}{rl}\frac{\partial E}{\partial w_{ij}^L}& =\frac{\partial E}{\partial {\mathbf{y}}^L}.\frac{\partial {\mathbf{y}}^L}{\partial w_{ij}^L}\\ & =\frac{\partial E}{\partial ({\mathbf{y}}^L{)}^{\mathrm{T}}}.\frac{\partial ({\mathbf{y}}^L{)}^{\mathrm{T}}}{\partial {\mathbf{y}}^L}.\frac{\partial {\mathbf{y}}^L}{\partial w_{ij}^L}.\\ & =\left(\begin{array}{c}{y}_1^L-t_1,\cdots ,y_{sL}^L-t_{sL}\end{array}\right).\mathbf{I}(i).z_i^{L-1}\\ & \\ \frac{\partial E}{\partial w_{ij}^{L-1}}& =\frac{\partial E}{\partial {\mathbf{y}}^L}.\frac{\partial {\mathbf{y}}^L}{\partial {\mathbf{y}}^{L-1}}.\frac{\partial {\mathbf{y}}^{L-1}}{\partial w_{ij}^{L-1}}\\ & =\left(\begin{array}{c}{y}_1^L-t_1,\cdots ,y_{sL}^L-t_{sL}\end{array}\right).\left(\begin{array}{ccc}{f}_1^L{w}_{11}^L& \cdots & f_1^L{w}_{(s(L-1))1}\\ ⋮& & ⋮\\ f_{sL}^L{w}_{(sL)1}^L& \cdots & f_{sL}^L{w}_{(sL)(s(L-1))}^L\end{array}\right).\mathbf{I}(i).z_i^{l-1}\\ & =\sum _{k=1}^{sl}(y_k^L-t_k)f_1^L{w}_{ki}^L{z}_j^{L-1}\end{array}$$
另一种定义方法
$$\begin{array}{rl}\frac{\partial E}{\partial w_{ij}^L}& =\frac{\partial E}{\partial y_i^L}.\frac{\partial y_i^L}{\partial w_{ij}^L}\\ & =(y_i^L-h_i)z_i^{L-1}\\ & \\ \frac{\partial E}{\partial w_{ij}^{L-1}}& =\frac{\partial E}{\partial {\mathbf{y}}^L}.\frac{\partial {\mathbf{y}}^L}{\partial y_i^{L-1}}.\frac{\partial y_i^{L-1}}{\partial w_{ij}^{L-1}}\\ & =\left(\begin{array}{c}{y}_1^L-t_1,\cdots ,y_{sL}^L-t_{sL}\end{array}\right).\left(\begin{array}{c}{f}_1^L{w}_{1i}^L\\ f_2^L{w}_{2i}^L\\ ⋮\\ f_{(sL)}^L{w}_{(sL)i}^L\end{array}\right).z_i^{L-1}\\ & =\sum _{k=1}^{sl}(y_k^L-t_k)f_1^L{w}_{ki}^L{z}_j^{L-1}\\ & \\ {\delta }_i^{l-1}& =\frac{\partial E}{\partial y_i^{l-1}}\\ & =\frac{\partial E}{\partial {\mathbf{y}}^l}.\frac{\partial {\mathbf{y}}^l}{\partial y_i^{l-1}}\\ & =\left(\begin{array}{c}{\delta }_1^l,{\delta }_2^l,\cdots ,{\delta }_{sl}^l\end{array}\right).\left(\begin{array}{c}{f}_1^l{w}_{1i}^l\\ f_2^l{w}_{2i}^l\\ ⋮\\ f_{(sl)}^l{w}_{(sl)i}^l\end{array}\right)\\ & =\sum _{k=1}^{sl}{\delta }_k^l{f}_k^l{w}_{(sl)i}^l\\ & \\ \frac{\partial E}{\partial w_{ij}^L}& =\frac{\partial E}{\partial y_i^L}.\frac{\partial y_i^L}{\partial w_{ij}^L}\\ & ={\delta }_i^L{z}_i^{L-1}\\ & =(y_i^L-h_i)z_i^{L-1}\\ & \\ \frac{\partial E}{\partial w_{ij}^l}& =\frac{\partial E}{\partial {\mathbf{y}}^l}.\frac{\partial y_i^l}{\partial w_{ij}^l}\\ & ={\delta }_i^l{z}_i^{l-1}\\ & =\sum _{k=1}^{s(l+1)}{\delta }_k^{(l+1)}{f}_k^{l+1}{w}_{ki}^{l+1}{z}_j^{l-1}\end{array}$$