【西瓜书笔记】9. EM算法（下）

接上一篇

$$Q\left(\theta \mid {\theta }^{(i)}\right)=\sum _{j=1}^N\left\{{\mu }_j^{(i+1)}\ln \left[\pi p^{y_j}(1-p{)}^{1-y_j}\right]+\left(1-{\mu }_j^{(i+1)}\right)\ln \left[(1-\pi )q^{y_j}(1-q{)}^{1-y_j}\right]\right\}$$

现在有了Q函数，我们可以进一步做M步：求使得Q函数达到极大的${\theta }^{(i+1)}=\left({\pi }^{(i+1)},p^{(i+1)},q^{(i+1)}\right)$。我们对Q函数关于$\pi$求一阶偏导数，并令一阶偏导数等于0

$$\begin{array}{rl}\frac{\partial Q\left(\theta \mid {\theta }^{(i)}\right)}{\partial \pi }& =\sum _{j=1}^N\frac{\partial }{\partial \pi }\left\{{\mu }_j^{(i+1)}\ln \left[\pi p^{y_j}(1-p{)}^{1-y_j}\right]+\left(1-{\mu }_j^{(i+1)}\right)\ln \left[(1-\pi )q^{y_j}(1-q{)}^{1-y_j}\right]\right\}\\ & =\sum _{j=1}^N\left\{{\mu }_j^{(i+1)}\frac{p^{y_j}(1-p{)}^{1-y_j}}{\pi p^{y_j}(1-p{)}^{1-y_j}}+\left(1-{\mu }_j^{(i+1)}\right)\frac{-q^{y_j}(1-q{)}^{1-y_j}}{(1-\pi )q^{y_j}(1-q{)}^{1-y_j}}\right\}\\ & =\sum _{j=1}^N\left\{\frac{{\mu }_j^{(i+1)}(1-\pi )p^{y_j}(1-p{)}^{1-y_j}{q}^{y_j}(1-q{)}^{1-y_j}}{\pi (1-\pi )p^{y_j}(1-p{)}^{1-y_j}{q}^{y_j}(1-q{)}^{1-y_j}}+\frac{\left({\mu }_j^{(i+1)}-1\right)\pi p^{y_j}(1-p{)}^{1-y_j}{q}^{y_j}(1-q{)}^{1-y_j}}{\pi (1-\pi )p^{y_j}(1-p{)}^{1-y_j}{q}^{y_j}(1-q{)}^{1-y_j}}\right\}\\ & =\sum _{j=1}^N\left\{\frac{{\mu }_j^{(i+1)}{p}^{y_j}(1-p{)}^{1-y_j}{q}^{y_j}(1-q{)}^{1-y_j}-\pi p^{y_j}(1-p{)}^{1-y_j}{q}^{y_j}(1-q{)}^{1-y_j}}{\pi (1-\pi )p^{y_j}(1-p{)}^{1-y_j}{q}^{y_j}(1-q{)}^{1-y_j}}\right\}\\ & =\sum _{j=1}^N\left[\frac{{\mu }_j^{(i+1)}-\pi }{\pi (1-\pi )}\right]\\ & =\frac{{\sum }_{j=1}^N{\mu }_j^{(i+1)}-{\sum }_{j=1}^N\pi }{\pi (1-\pi )}\\ & =\frac{{\sum }_{j=1}^N{\mu }_j^{(i+1)}-N\pi }{\pi (1-\pi )}\end{array}$$
所以

$$\begin{array}{c}\frac{\partial Q\left(\theta \mid {\theta }^{(i)}\right)}{\partial \pi }=\frac{{\sum }_{j=1}^N{\mu }_j^{(i+1)}-N\pi }{\pi (1-\pi )}=0\\ \sum _{j=1}^N{\mu }_j^{(i+1)}-N\pi =0\\ N\pi =\sum _{j=1}^N{\mu }_j^{(i+1)}\\ \pi =\frac{1}{N}\sum _{j=1}^N{\mu }_j^{(i+1)}\Rightarrow {\pi }^{(i+1)}=\frac{1}{N}\sum _{j=1}^N{\mu }_j^{(i+1)}\end{array}$$
对Q函数关于p求偏导

$$\begin{array}{rl}\frac{\partial Q\left(\theta \mid {\theta }^{(i)}\right)}{\partial p}& =\sum _{j=1}^N\frac{\partial }{\partial p}\left\{{\mu }_j^{(i+1)}\ln \left[\pi p^{y_j}(1-p{)}^{1-y_j}\right]+\left(1-{\mu }_j^{(i+1)}\right)\ln \left[(1-\pi )q^{y_j}(1-q{)}^{1-y_j}\right]\right\}\\ & =\sum _{j=1}^N\frac{\partial }{\partial p}\left\{{\mu }_j^{(i+1)}\ln \left[\pi p^{y_j}(1-p{)}^{1-y_j}\right]\right\}\\ & =\sum _{j=1}^N\frac{\partial }{\partial p}\left\{{\mu }_j^{(i+1)}\left[\ln \pi +y_j\ln p+\left(1-y_j\right)\ln (1-p)\right]\right\}\\ & =\sum _{j=1}^N\frac{\partial }{\partial p}\left\{{\mu }_j^{(i+1)}\ln \pi +{\mu }_j^{(i+1)}{y}_j\ln p+{\mu }_j^{(i+1)}\left(1-y_j\right)\ln (1-p)\right\}\\ & =\sum _{j=1}^N\frac{\partial }{\partial p}\left\{{\mu }_j^{(i+1)}{y}_j\ln p+{\mu }_j^{(i+1)}\left(1-y_j\right)\ln (1-p)\right\}\\ & =\sum _{j=1}^N\left\{\frac{{\mu }_j^{(i+1)}{y}_j}{p}+\frac{(-1)\cdot {\mu }_j^{(i+1)}\left(1-y_j\right)}{(1-p)}\right\}\\ & =\sum _{j=1}^N\frac{{\mu }_j^{(i+1)}{y}_j}{p}-\sum _{j=1}^N\frac{{\mu }_j^{(i+1)}\left(1-y_j\right)}{(1-p)}\\ & =\frac{{\sum }_{j=1}^N{\mu }_j^{(i+1)}{y}_j}{p}-\frac{{\sum }_{j=1}^N{\mu }_j^{(i+1)}\left(1-y_j\right)}{(1-p)}\end{array}$$

等于0有

$$\begin{array}{c}\frac{\partial Q\left(\theta \mid {\theta }^{(i)}\right)}{\partial p}=\frac{{\sum }_{j=1}^N{\mu }_j^{(i+1)}{y}_j}{p}-\frac{{\sum }_{j=1}^N{\mu }_j^{(i+1)}\left(1-y_j\right)}{(1-p)}=0\\ \frac{{\sum }_{j=1}^N{\mu }_j^{(i+1)}{y}_j}{p}=\frac{{\sum }_{j=1}^N{\mu }_j^{(i+1)}\left(1-y_j\right)}{(1-p)}\\ (1-p)\sum _{j=1}^N{\mu }_j^{(i+1)}{y}_j=p\sum _{j=1}^N{\mu }_j^{(i+1)}\left(1-y_j\right)\\ \sum _{j=1}^N{\mu }_j^{(i+1)}{y}_j-p\sum _{j=1}^N{\mu }_j^{(i+1)}{y}_j=p\sum _{j=1}^N{\mu }_j^{(i+1)}-p\sum _{j=1}^N{\mu }_j^{(i+1)}{y}_j\\ \sum _{j=1}^N{\mu }_j^{(i+1)}{y}_j=p\sum _{j=1}^N{\mu }_j^{(i+1)}\\ p=\frac{{\sum }_{j=1}^N{\mu }_j^{(i+1)}{y}_j}{{\sum }_{j=1}^N{\mu }_j^{(i+1)}}\Rightarrow p^{(i+1)}=\frac{{\sum }_{j=1}^N{\mu }_j^{(i+1)}{y}_j}{{\sum }_{j=1}^N{\mu }_j^{(i+1)}}\end{array}$$
此即为式9.7

对Q函数关于q求导，并令一阶导等于0：
$$\begin{array}{rl}\frac{\partial Q\left(\theta \mid {\theta }^{(i)}\right)}{\partial q}& =\sum _{j=1}^N\frac{\partial }{\partial q}\left\{{\mu }_j^{(i+1)}\ln \left[\pi p^{y_j}(1-p{)}^{1-y_j}\right]+\left(1-{\mu }_j^{(i+1)}\right)\ln \left[(1-\pi )q^{y_j}(1-q{)}^{1-y_j}\right]\right\}\\ & =\sum _{j=1}^N\frac{\partial }{\partial q}\left\{\left(1-{\mu }_j^{(i+1)}\right)\ln \left[(1-\pi )q^{y_j}(1-q{)}^{1-y_j}\right]\right\}\\ & =\sum _{j=1}^N\frac{\partial }{\partial q}\left\{\left(1-{\mu }_j^{(i+1)}\right)\left[\ln (1-\pi )+y_j\ln q+\left(1-y_j\right)\ln (1-q)\right]\right\}\\ & =\sum _{j=1}^N\frac{\partial }{\partial q}\left\{\left(1-{\mu }_j^{(i+1)}\right)\ln (1-\pi )+\left(1-{\mu }_j^{(i+1)}\right)y_j\ln q+\left(1-{\mu }_j^{(i+1)}\right)\left(1-y_j\right)\ln (1-q)\right\}\\ & =\sum _{j=1}^N\frac{\partial }{\partial q}\left\{\left(1-{\mu }_j^{(i+1)}\right)y_j\ln q+\left(1-{\mu }_j^{(i+1)}\right)\left(1-y_j\right)\ln (1-q)\right\}\\ & =\sum _{j=1}^N\left\{\frac{\left(1-{\mu }_j^{(i+1)}\right)y_j}{q}+\frac{(-1)\cdot \left(1-{\mu }_j^{(i+1)}\right)\left(1-y_j\right)}{(1-q)}\right\}\\ & =\sum _{j=1}^N\frac{\left(1-{\mu }_j^{(i+1)}\right)y_j}{q}-\sum _{j=1}^N\frac{\left(1-{\mu }_j^{(i+1)}\right)\left(1-y_j\right)}{(1-q)}\\ & =\frac{{\sum }_{j=1}^N\left(1-{\mu }_j^{(i+1)}\right)y_j}{q}-\frac{{\sum }_{j=1}^N\left(1-{\mu }_j^{(i+1)}\right)\left(1-y_j\right)}{(1-q)}\end{array}$$

所以
$$\begin{array}{c}\frac{\partial Q\left(\theta \mid {\theta }^{(i)}\right)}{\partial q}=\frac{{\sum }_{j=1}^N\left(1-{\mu }_j^{(i+1)}\right)y_j}{q}-\frac{{\sum }_{j=1}^N\left(1-{\mu }_j^{(i+1)}\right)\left(1-y_j\right)}{(1-q)}=0\\ \frac{{\sum }_{j=1}^N\left(1-{\mu }_j^{(i+1)}\right)y_j}{q}=\frac{{\sum }_{j=1}^N\left(1-{\mu }_j^{(i+1)}\right)\left(1-y_j\right)}{(1-q)}\\ (1-q)\sum _{j=1}^N\left(1-{\mu }_j^{(i+1)}\right)y_j=q\sum _{j=1}^N\left(1-{\mu }_j^{(i+1)}\right)\left(1-y_j\right)\\ \sum _{j=1}^N\left(1-{\mu }_j^{(i+1)}\right)y_j-q\sum _{j=1}^N\left(1-{\mu }_j^{(i+1)}\right)y_j=q\sum _{j=1}^N\left(1-{\mu }_j^{(i+1)}\right)-q\sum _{j=1}^N\left(1-{\mu }_j^{(i+1)}\right)y_j\\ \sum _{j=1}^N\left(1-{\mu }_j^{(i+1)}\right)y_j=q\sum _{j=1}^N\left(1-{\mu }_j^{(i+1)}\right)\\ q=\frac{{\sum }_{j=1}^N\left(1-{\mu }_j^{(i+1)}\right)y_j}{{\sum }_{j=1}^N\left(1-{\mu }_j^{(i+1)}\right)}\Rightarrow q^{(i+1)}=\frac{{\sum }_{j=1}^N\left(1-{\mu }_j^{(i+1)}\right)y_j}{{\sum }_{j=1}^N\left(1-{\mu }_j^{(i+1)}\right)}\end{array}$$
此即为式9.8。