高斯判别分析（Gaussian Discriminative Analysis）

  高斯判别分析(GDA)是经典的生成学习模型，也是一种监督分类学习算法。
  假设有样本集 D = { ( x 1 , y 1 ) , ( x 2 , y 2 ) , . . . , ( x n , y n ) } D=\{(x_1,y_1),(x_2,y_2),...,(x_n,y_n)\} D={(x1​,y1​),(x2​,y2​),...,(xn​,yn​)}，其中 x i ∈ R d , y i ∈ { 0 , 1 } x_i \in R^d,y_i \in \{0,1\} xi​∈Rd,yi​∈{0,1}。高斯判别分析作为生成学习算法，同样也是对联合概率$P(x,y)$建模，在GDA模型中首先假设：
$$\begin{gathered} y\sim Bernoulli(\varphi ) \\ x|y=0\sim N({\mu }_0,\Sigma ) \\ x|y=1\sim N({\mu }_1,\Sigma ) \end{gathered}$$
其概率分布：
$$\begin{gathered} p(y)={\varphi }^y(1-\varphi {)}^{1-y} \\ p(x|y=0)=\frac{1}{(2\pi {)}^{\frac{d}{2}}|\Sigma {|}^{\frac{1}{2}}}exp\left(-\frac{1}{2}(x-{\mu }_0{)}^T{\Sigma }^{-1}(x-{\mu }_0)\right) \\ p(x|y=1)=\frac{1}{(2\pi {)}^{\frac{d}{2}}|\Sigma {|}^{\frac{1}{2}}}exp\left(-\frac{1}{2}(x-{\mu }_1{)}^T{\Sigma }^{-1}(x-{\mu }_1)\right) \end{gathered}$$
在样本集D上的对数似然函数：
$$\begin{array}{rl}l(\varphi ,{\mu }_0,{\mu }_1,\Sigma )& =log\prod _{i=1}^{m}P(x_i,y_i;\varphi ,{\mu }_0,{\mu }_1,\Sigma )\\ & =log\prod _{i=1}^{m}P(x_i|y_i;{\mu }_0,{\mu }_1,\Sigma )P(y_i;\varphi )\\ & =\sum _{i=1}^{m}logP(x_i|y_i;{\mu }_0,{\mu }_1,\Sigma )+logP(y_i;\varphi )\\ & =\sum _{i=1}^{m}logP(x_i|y_i=0;{\mu }_0,\Sigma {)}^{1-y_i}P(x_i|y_i=1;{\mu }_1,\Sigma {)}^{y_i}+logP(y_i;\varphi )\\ & =\sum _{i=1}^m(1-y_i)logP(x_i|y_i=0;{\mu }_0,\Sigma )+y_{i}logP(x_i|y_i=1;{\mu }_1,\Sigma )+logP(y_i;\varphi )\\ & =\sum _{i=1}^m(1-y_i)[-\frac{d}{2}log2\pi -\frac{1}{2}log|\Sigma |-\frac{1}{2}(x-{\mu }_0{)}^T{\Sigma }^{-1}(x-{\mu }_0)]\\ & +y_i[-\frac{d}{2}log2\pi -\frac{1}{2}log|\Sigma |-\frac{1}{2}(x-{\mu }_1{)}^T{\Sigma }^{-1}(x-{\mu }_1)]+log{\varphi }^y(1-\varphi {)}^{1-y}\end{array}$$
在计算似然函数的最大值我们先了解几个公式：

$$\begin{gathered} trABC=trCAB=trBCA \\ \frac{\partial trAX}{\partial X}=\frac{\partial trXA}{\partial X}=A^T \\ \frac{\partial u^{T}v}{\partial x}=\frac{\partial uv}{\partial x}=\frac{\partial u}{\partial x}v+\frac{\partial v}{\partial x}u \\ \frac{\partial log|X|}{\partial X}=\frac{1}{|X|}|X|(X^{-1}{)}^T \\ \frac{\partial |X|}{\partial X}=(X^{-1}{)}^T \\ \frac{\partial tr{X}^{-1}A}{\partial X}=-(X^{-1}{)}^T{A}^T(X^{-1}{)}^T \end{gathered}$$

我们通过最大似然函数估计参数：
$$\begin{array}{rl}\frac{\partial l(\varphi ,{\mu }_0,{\mu }_1,\Sigma )}{\partial \varphi }& =\frac{\partial {\sum }_{i=1}^{m}log{\varphi }^{y_i}(1-\varphi {)}^{1-y_i}}{\partial \varphi }\\ & =\frac{\partial {\sum }_{i=1}^m{y}_{i}log\varphi +(1-y_i)log(1-\varphi )}{\partial \varphi }\\ & =\sum _{i=1}^m\frac{y_i}{\varphi }-\frac{1-y_i}{1-\varphi }=\sum _{i=1}^m\frac{y_i-\varphi }{\varphi (1-\varphi )}=0\\ & \Rightarrow \sum _{i=1}^m{y}_i-\varphi =0\Rightarrow \sum _{i=1}^m{y}_i=\sum _{i=1}^m\varphi =m\varphi \\ \varphi & =\frac{{\sum }_{i=1}^{m}I(y_i=1)}{m}\end{array}$$
$$\begin{array}{rl}\frac{\partial l(\varphi ,{\mu }_0,{\mu }_1,\Sigma )}{\partial {\mu }_0}& =\frac{\partial {\sum }_{i=1}^m(1-y_i)[-\frac{1}{2}(x_i-{\mu }_0{)}^T{\Sigma }^{-1}(x_i-{\mu }_0)]}{\partial {\mu }_0}\\ & =\sum _{i=1}^m-\frac{1}{2}(1-y_i)[\frac{\partial (x_i-{\mu }_0)}{\partial {\mu }_0}{\Sigma }^{-1}(x_i-{\mu }_0)+\frac{\partial {\Sigma }^{-1}(x_i-{\mu }_0)}{\partial {\mu }_0}(x_i-{\mu }_0)]\\ & =\sum _{i=1}^m-\frac{1}{2}(1-y_i)[-{\Sigma }^{-1}(x_i-{\mu }_0)-({\Sigma }_{-1}{)}^T(x_i-{\mu }_0)]\\ & =\sum _{i=1}^m(1-y_i){\Sigma }^{-1}(x_i-{\mu }_0)=0\\ & \Rightarrow \sum _{i=1}^m(1-y_i)\Sigma {\Sigma }^{-1}(x_i-{\mu }_0)=0\Sigma \Rightarrow \sum _{i=1}^m(1-y_i)(x_i-{\mu }_0)=0\\ {\mu }_0& =\frac{{\sum }_{i=1}^{m}I(y_i=0)x_i}{m}\end{array}$$
$$\begin{array}{rl}\frac{\partial l(\varphi ,{\mu }_0,{\mu }_1,\Sigma )}{\partial {\mu }_1}& =\frac{\partial {\sum }_{i=1}^m{y}_i[-\frac{1}{2}(x_i-{\mu }_1{)}^T{\Sigma }^{-1}(x_i-{\mu }_1)]}{\partial {\mu }_1}\\ & =\sum _{i=1}^m-\frac{1}{2}{y}_i[\frac{\partial (x_i-{\mu }_1)}{\partial {\mu }_1}{\Sigma }^{-1}(x_i-{\mu }_1)+\frac{\partial {\Sigma }^{-1}(x_i-{\mu }_1)}{\partial {\mu }_1}(x_i-{\mu }_1)]\\ & =\sum _{i=1}^m-\frac{1}{2}{y}_i[-{\Sigma }^{-1}(x_i-{\mu }_1)-({\Sigma }^{-1}{)}^T(x_i-{\mu }_1)]\\ & =\sum _{i=1}^m{y}_i{\Sigma }^{-1}(x_i-{\mu }_1)=0\\ & \Rightarrow \sum _{i=1}^m{y}_i\Sigma {\Sigma }^{-1}(x_i-{\mu }_1)=0\Sigma \Rightarrow \sum _{i=1}^m{y}_i(x_i-{\mu }_1)=0\\ {\mu }_1& =\frac{{\sum }_{i=1}^{m}I(y_i=1)x_i}{m}\end{array}$$

$$\begin{array}{rl}\frac{\partial l(\varphi ,{\mu }_0,{\mu }_1,\Sigma )}{\partial \Sigma }& =\frac{\partial {\sum }_{i=1}^m{y}_i[-\frac{1}{2}(x_i-{\mu }_1{)}^T{\Sigma }^{-1}(x_i-{\mu }_1)]}{\partial \Sigma }+\frac{\partial {\sum }_{i=1}^m(1-y_i)[-\frac{1}{2}(x_i-{\mu }_0{)}^T{\Sigma }^{-1}(x_i-{\mu }_0)]}{\partial \Sigma }+\frac{\partial {\sum }_{i=1}^m-\frac{1}{2}log|\Sigma |}{\partial \Sigma }\\ & =\sum _{i=1}^m{y}_i\frac{\partial tr[-\frac{1}{2}(x_i-{\mu }_1{)}^T{\Sigma }^{-1}(x_i-{\mu }_1)]}{\partial \Sigma }+(1-y_i)\frac{\partial tr[-\frac{1}{2}(x_i-{\mu }_0{)}^T{\Sigma }^{-1}(x_i-{\mu }_0)]}{\partial \Sigma }+\sum _{i=1}^m\frac{\partial (-\frac{1}{2}log|\Sigma |)}{\partial \Sigma }\\ & =\sum _{i=1}^m-\frac{1}{2}{y}_i\frac{\partial tr[{\Sigma }^{-1}(x_i-{\mu }_1)(x_i-{\mu }_1{)}^T]}{\partial \Sigma }-\frac{1}{2}(1-y_i)\frac{\partial tr[{\Sigma }^{-1}(x_i-{\mu }_0)(x_i-{\mu }_0{)}^T]}{\partial \Sigma }+\sum _{i=1}^m-\frac{1}{2}\frac{1}{|\Sigma |}|\Sigma |({\Sigma }^{-1}{)}^T\\ & =\sum _{i=1}^m-\frac{1}{2}{y}_i[-({\Sigma }^{-1}{)}^T((x_i-{\mu }_1)(x_i-{\mu }_1{)}^T{)}^T({\Sigma }^{-1}{)}^T]-\frac{1}{2}(1-y_i)[-({\Sigma }^{-1}{)}^T((x_i-{\mu }_0)(x_i-{\mu }_0{)}^T{)}^T({\Sigma }^{-1}{)}^T]-\frac{1}{2}m({\Sigma }^{-1}{)}^T\\ & =\sum _{i=1}^m-\frac{1}{2}{y}_i[-{\Sigma }^{-1}(x_i-{\mu }_1)(x_i-{\mu }_1{)}^T{\Sigma }^{-1}]-\frac{1}{2}(1-y_i)[-{\Sigma }^{-1}(x_i-{\mu }_0)(x_i-{\mu }_0{)}^T{\Sigma }^{-1}]-\frac{1}{2}m{\Sigma }^{-1}=0\\ & \Rightarrow \sum _{i=1}^m-\frac{1}{2}{y}_i[-\Sigma {\Sigma }^{-1}(x_i-{\mu }_1)(x_i-{\mu }_1{)}^T\Sigma {\Sigma }^{-1}]-\frac{1}{2}(1-y_i)[-\Sigma {\Sigma }^{-1}(x_i-{\mu }_0)(x_i-{\mu }_0{)}^T\Sigma {\Sigma }^{-1}]-\frac{1}{2}m\Sigma {\Sigma }^{-1}\Sigma =\Sigma 0\Sigma \\ & \Rightarrow \sum _{i=1}^m-\frac{1}{2}{y}_i[-(x_i-{\mu }_1)(x_i-{\mu }_1{)}^T]-\frac{1}{2}(1-y_i)[-(x_i-{\mu }_0)(x_i-{\mu }_0{)}^T]-\frac{1}{2}m\Sigma =0\\ & \Rightarrow \sum _{i=1}^m{y}_i[(x_i-{\mu }_1)(x_i-{\mu }_1{)}^T]+(1-y_i)[(x_i-{\mu }_0)(x_i-{\mu }_0{)}^T]-m\Sigma =0\\ \Sigma & =\frac{{\sum }_{i=1}^m{y}_i[(x_i-{\mu }_1)(x_i-{\mu }_1{)}^T]+(1-y_i)[(x_i-{\mu }_0)(x_i-{\mu }_0{)}^T]}{m}\end{array}$$
综上，我们有：
$$\begin{gathered} \varphi =\frac{{\sum }_{i=1}^{m}I(y_i=1)}{m} \\ {\mu }_0=\frac{{\sum }_{i=1}^{m}I(y_i=0)x_i}{m} \\ {\mu }_1=\frac{{\sum }_{i=1}^{m}I(y_i=1)x_i}{m} \\ \Sigma =\frac{{\sum }_{i=1}^m{y}_i[(x_i-{\mu }_1)(x_i-{\mu }_1{)}^T]+(1-y_i)[(x_i-{\mu }_0)(x_i-{\mu }_0{)}^T]}{m} \end{gathered}$$