线性回归解析解推导以及实现

公式1：

$$w_1^{\ast }=\frac{{\sum }_i{y}_i(x_i-\frac{1}{n}{\sum }_i{x}_i)}{{\sum }_i{x}_i^2-\frac{1}{n}({\sum }_i{x}_i{)}^2}$$
推导上式，给定以下条件：
$$\{\begin{array}{l}{w}_0^{\ast }=(\frac{1}{n}{\sum }_i{y}_i)-w_1^{\ast }(\frac{1}{n}{\sum }_i{x}_i)\\ w_1^{\ast }=-{\sum }_i{x}_i(w_0^{\ast }-y_i)/{\sum }_i{x}_i^2\end{array}$$
这里相当于求解方程组，消去 $w_0^{\ast }$ 即可
$$w_1^{\ast }=\frac{-w_0^{\ast }{\sum }_i{x}_i+{\sum }_i{x}_i{y}_i}{{\sum }_i{x}_i^2}\text{\hspace{1em}(1)}$$

$$\begin{array}{rl}& \left(\frac{{\sum }_i{x}_i}{{\sum }_i{x}_i^2}\right)\ast w_0^{\ast }=\left(\frac{{\sum }_i{x}_i}{{\sum }_i{x}_i^2}\right)\ast \left((\frac{1}{n}\sum _i{y}_i)-w_1^{\ast }(\frac{1}{n}\sum _i{x}_i)\right)\\ & \left(\frac{{\sum }_i{x}_i}{{\sum }_i{x}_i^2}\right)\ast w_0^{\ast }=\frac{\left({\sum }_i{x}_i\right)\ast \left(\frac{1}{n}{\sum }_i{y}_i\right)}{{\sum }_i{x}_i^2}-\frac{\left({\sum }_i{x}_i\right)\ast \left(-w_1^{\ast }(\frac{1}{n}{\sum }_i{x}_i)\right)}{{\sum }_i{x}_i^2}\end{array}\text{\hspace{1em}(2)}$$

---

$$\begin{array}{rl}(1)+(2)& ⟹w_1^{\ast }-\frac{w_1^{\ast }\left(\frac{1}{n}{\left({\sum }_i{x}_i\right)}^2\right)}{{\sum }_i{x}_i^2}+\frac{\frac{1}{n}{\sum }_i{x}_i{\sum }_i{y}_i}{{\sum }_i{x}_i^2}=\frac{{\sum }_i{x}_i{y}_i}{{\sum }_i{x}_i^2}\\ & ⟹w_1^{\ast }\left(\frac{{\sum }_i{x}_i^2-\frac{1}{n}{\left({\sum }_i{x}_i\right)}^2}{{\sum }_i{x}_i^2}\right)=\frac{{\sum }_i{x}_i{y}_i-\frac{1}{n}{\sum }_i{x}_i{\sum }_i{y}_i}{{\sum }_i{x}_i^2}\\ & ⟹w_1^{\ast }=\frac{{\sum }_i{y}_i(x_i-\frac{1}{n}{\sum }_i{x}_i)}{{\sum }_i{x}_i^2-\frac{1}{n}({\sum }_i{x}_i{)}^2}=\frac{{\sum }_i{x}_i(y_i-\frac{1}{n}{\sum }_i{y}_i)}{{\sum }_i{x}_i^2-\frac{1}{n}({\sum }_i{x}_i{)}^2}\end{array}$$
推导完毕.

公式2

$$\hat{\mathrm{w}}=({\mathrm{X}}^{\mathrm{T}}\mathrm{X}{)}^{-1}{\mathrm{X}}^{\mathrm{T}}\mathbf{y}$$
推导上式，给定以下条件：
$$\underset{w_0,w_1}{\mathrm{argmin}}L(\hat{w})=\Vert \mathrm{Y}-\mathrm{X}\hat{\mathrm{w}}{\Vert }^2$$

先化简：
$\begin{array}{rl}\Vert \mathrm{X}\hat{\mathrm{w}}-\mathrm{Y}{\Vert }^2& =(\mathrm{X}\hat{\mathrm{w}}-\mathrm{Y}{)}^{\mathrm{T}}(\mathrm{X}\hat{\mathrm{w}}-\mathrm{Y})\\ & =({\hat{\mathrm{w}}}^{\mathrm{T}}{\mathrm{X}}^{\mathrm{T}}-{\mathrm{Y}}^{\mathrm{T}})(\mathrm{X}\hat{\mathrm{w}}-\mathrm{Y})\\ & ={\hat{\mathrm{w}}}^{\mathrm{T}}{\mathrm{X}}^{\mathrm{T}}\mathrm{X}\hat{\mathrm{w}}-{\hat{\mathrm{w}}}^{\mathrm{T}}{\mathrm{X}}^{\mathrm{T}}\mathrm{Y}-{\mathrm{Y}}^{\mathrm{T}}\mathrm{X}\hat{\mathrm{w}}+{\mathrm{Y}}^{\mathrm{T}}\mathrm{Y}\end{array}$

---

对 $\hat{\mathrm{w}}$ 求导使得下式为0：
$$\begin{array}{rl}\frac{\partial \left(\Vert \mathrm{X}\hat{\mathrm{w}}-\mathrm{Y}{\Vert }^2\right)}{\partial \hat{\mathrm{w}}}& =\frac{\partial \left({\hat{\mathrm{w}}}^{\mathrm{T}}{\mathrm{X}}^{\mathrm{T}}\mathrm{X}\hat{\mathrm{w}}-{\hat{\mathrm{w}}}^{\mathrm{T}}{\mathrm{X}}^{\mathrm{T}}\mathrm{Y}-{\mathrm{Y}}^{\mathrm{T}}\mathrm{X}\hat{\mathrm{w}}+{\mathrm{Y}}^{\mathrm{T}}\mathrm{Y}\hat{\mathrm{w}}\right)}{\partial \hat{\mathrm{w}}}=0\end{array}$$

---

以下是矩阵求导公式：
$$\frac{\partial \left({\hat{\mathrm{w}}}^{\mathrm{T}}{\mathrm{X}}^{\mathrm{T}}\mathrm{X}\hat{\mathrm{w}}\right)}{\partial \hat{\mathrm{w}}}=2{\mathrm{X}}^{\mathrm{T}}\mathrm{X}\hat{\mathrm{w}}$$

$$\frac{\partial \left({\hat{\mathrm{w}}}^{\mathrm{T}}{\mathrm{X}}^{\mathrm{T}}\mathrm{Y}\right)}{\partial \hat{\mathrm{w}}}={\mathrm{X}}^{\mathrm{T}}\mathrm{Y}$$

$$\frac{\partial \left({\mathrm{Y}}^{\mathrm{T}}\mathrm{X}\hat{\mathrm{w}}\right)}{\partial \hat{\mathrm{w}}}={\mathrm{X}}^{\mathrm{T}}\mathrm{Y}$$

$$\frac{\partial \left({\mathrm{Y}}^{\mathrm{T}}\mathrm{Y}\right)}{\partial \hat{\mathrm{w}}}=0$$

---

推导：
$$\begin{array}{rl}& 2{\mathrm{X}}^{\mathrm{T}}\mathrm{X}\hat{\mathrm{w}}-{\mathrm{X}}^{\mathrm{T}}\mathrm{Y}-{\mathrm{X}}^{\mathrm{T}}\mathrm{Y}=0\\ ⟹& 2({\mathrm{X}}^{\mathrm{T}}\mathrm{X}\hat{\mathrm{w}}-{\mathrm{X}}^{\mathrm{T}}\mathrm{Y})=0\\ ⟹& {\mathrm{X}}^{\mathrm{T}}\mathrm{X}\hat{\mathrm{w}}-{\mathrm{X}}^{\mathrm{T}}\mathrm{Y}=0\\ ⟹& ({\mathrm{X}}^{\mathrm{T}}\mathrm{X})\hat{\mathrm{w}}={\mathrm{X}}^{\mathrm{T}}\mathrm{Y}\\ ⟹& \hat{\mathrm{w}}=({\mathrm{X}}^{\mathrm{T}}\mathrm{X}{)}^{-1}{\mathrm{X}}^{\mathrm{T}}\mathrm{Y}\end{array}$$

代码实现(Python)

import numpy as np
import matplotlib.pyplot as plt
import time


def get_fake_data(iter):
    X = np.random.rand(iter) * 20
    noise = np.random.randn(iter)
    y = 0.5 * X + noise
    plt.scatter(X, y)
    return X, y


def equation1(X_train, y_train):
    stat_time = time.time()
    num_instances = X_train.shape[0]
    w1 = np.sum(y_train * (X_train - np.sum(X_train) / num_instances)) / \
         (np.sum(X_train ** 2) - (np.sum(X_train) ** 2 / num_instances))
    w0 = np.sum(y_train) / num_instances - w1 * (np.sum(X_train) / num_instances)
    end_time = time.time()
    W = np.array((w1, w0))
    return W, (end_time - stat_time)


def equation2(X_train, y_train):
    stat_time = time.time()
    ones = np.ones(X_train.shape[0])
    X = np.column_stack((X_train.reshape(X_train.shape[0], 1), ones))
    W = (np.linalg.inv((X.T).dot(X)).dot(X.T)).dot(y_train)
    end_time = time.time()
    return W, (end_time - stat_time)


def equation1_test(X, y):
    print("公式1 : ")
    numInstances = X.shape[0]
    train_test_split = int(numInstances * 0.7)
    X_train, y_train = X[:train_test_split], y[:train_test_split]
    X_test, y_test = X[train_test_split:], y[train_test_split:]
    W, spend_time = equation1(X_train, y_train)

    # 画图
    ones = np.ones(X_test.shape[0])
    X_test = np.column_stack((X_test.reshape(X_test.shape[0], 1), ones))
    y_predict = X_test * W
    plt.plot(X_test, y_predict, color='#3479f7')

    # 输出
    print("权重 : ", W)
    print("运行时间 : ", spend_time)


def equation2_test(X, y):
    print("公式2 : ")
    numInstances = X.shape[0]
    train_test_split = int(numInstances * 0.7)
    X_train, y_train = X[:train_test_split], y[:train_test_split]
    X_test, y_test = X[train_test_split:], y[train_test_split:]
    W, spend_time = equation2(X_train, y_train)

    # 画图
    ones = np.ones(X_test.shape[0])
    X_test = np.column_stack((X_test.reshape(X_test.shape[0], 1), ones))
    y_predict = X_test * W
    plt.plot(X_test, y_predict, color='#9b59b6')

    print("权重 : ", W)
    print("运行时间 : ", spend_time)


if __name__ == '__main__':
    X, y = get_fake_data(100)
    equation1_test(X, y)
    equation2_test(X, y)
    plt.show()

公式1 : 
权重 :  [0.50045109 0.06868346]
运行时间 :  0.0
公式2 : 
权重 :  [0.50045109 0.06868346]
运行时间 :  0.37624287605285645