梯度下降求解线性回归

//导包
import numpy as np
import os
import matplotlib.pyplot as plt
%matplotlib inline 
//随机种子
np.random.seed(42)

//保存图像
Dir="."  //一个.表示统计目录    两个.表示父级目录
model_ID="linear_models"
//定义一个保存图像的函数
def save_fig(fig_id,tight_layout=True):
    path=os.path.join(Dir,"images",model_ID,fig_id+".png")  //指定保存图片的路径，当前目录下的images文件下的model_ID（linear_models）文件夹
    print("Saving figure",fig_id) //提示函数，正在保存图片
    plt.savefig(path,format="png",dpi=300) //保存图片（需要指定保存路径，保存格式和清晰度）
    
//把讨厌的警告信息过滤掉
import warnings
warnings.filterwarnings(action="ignore",message="^internal gelsd")
x=2*np.random.rand(100,1)  //生成训练数据（特征部分）
y=4+3*x+np.random.rand(100,1)  //生成训练数据（特征部分）
plt.plot(x,y,"b.") // 画图
plt.xlabel("$x_1$",fontsize=18)
plt.ylabel("$y$",rotation=0,fontsize=18)
plt.axis([0,2,0,15])
save_fig("gernated_data_plot") //保存图片
plt.show()

//添加新特征
x_b=np.c_[np.ones((100,1)),x]
//创建测试数据
x_new=np.array([[0],[2]])
x_new_b=np.c_[np.ones((2,1)),x_new]
from sklearn.linear_model import LinearRegression  //从sklearn包里导入线性回归模型
lin_reg=LinearRegression() //创建线性回归对象
lin_reg.fit(x,y) //拟合训练数据
lin_reg.intercept_,lin_reg.coef_ //输出截距，斜率
lin_reg.predict(x_new) //对测试集进行预测

一、用批量梯度下降求解线性回归

eta=0.1
n_iterations=1000
m=100
theta=np.random.randn(2,1)

for iteration in range(n_iterations):
    gradients=2/m*x_b.T.dot(x_b.dot(theta)-y)
    theta=theta-eta*gradients
   theta_path_bgd=[]
   
def plot_gradient_descent(theta,eta,theta_path=None):
    m=len(x_b)
    plt.plot(x,y,"b.")
    n_iterations=1000
    for iteration in range(n_iterations):
        if iteration<10:
            y_predict=x_new_b.dot(theta)
            style="b-"
            plt.plot(x_new,y_predict,style)
        gradients=2/m*x_b.T.dot(x_b.dot(theta)-y)
        theta=theta-eta*gradients
        if theta_path is not None:
            theta_path.append(theta)
    plt.xlabel("$x_1$",fontsize=18)
    plt.axis([0,2,0,15])
    plt.title(r"$\eta={}$".format(eta),fontsize=16)
np.random.seed(42)
theta=np.random.randn(2,1)
plt.figure(figsize=(10,4))
plt.subplot(131);plot_gradient_descent(theta,eta=0.02)
plt.ylabel("$y$",rotation=0,fontsize=18)
plt.subplot(132);plot_gradient_descent(theta,eta=0.1,theta_path=theta_path_bgd)
plt.subplot(133);plot_gradient_descent(theta,eta=0.5)

save_fig("gradient_decent_plot")
plt.show()

随机梯度下降

theta_path_sgd=[]
m=len(x_b)
np.random.seed(42)
n_epochs=50
theta=np.random.randn(2,1)
for epoch in range(n_epochs):
    for i in range(m):
        if epoch==0 and i <20:
            y_predict=x_new_b.dot(theta)
            style="b-"
            plt.plot(x_new,y_predict,style)
        random_index=np.random.randint(m)
        xi=x_b[random_index:random_index+1]
        yi=y[random_index:random_index+1]
        gradients=2*xi.T.dot(xi.dot(theta)-yi)
        eta=0.1
        theta=theta-eta*gradients
        theta_path_sgd.append(theta)
        
plt.plot(x,y,"b.")
plt.xlabel("$x_1$",fontsize=18)
plt.ylabel("$y$",rotation=0,fontsize=18)
plt.axis([0,2,0,15])
save_fig('sgd_plot')
plt.show()

Mini-batch gradint descent 小批量梯度下降

theta_path_mgd=[]
n_iterations=50
minibatch_size=20
np.random.seed(42)
theta=np.random.randn(2,1)

for epoch in range(n_iterations):
    shuffled_indices=np.random.permutation(m)
    x_b_shuffled=x_b[shuffled_indices]
    y_shuffled=y[shuffled_indices]
    for i in range(0,m,minibatch_size):
        xi=x_b_shuffled[i:i+minibatch_size]
        yi=y_shuffled[i:i+minibatch_size]
        gradients=2/minibatch_size*xi.T.dot(xi.dot(theta)-yi)
        eta=0.1
        theta=theta-eta*gradients
        theta_path_mgd.append(theta)

theta_path_bgd=np.array(theta_path_bgd)
theta_path_sgd=np.array(theta_path_sgd)
theta_path_mgd=np.array(theta_path_mgd)
plt.figure(figsize=(7,4))
plt.plot(theta_path_sgd[:,0],theta_path_sgd[:,1],"r-s",linewidth=1,label='Stochastic')
plt.plot(theta_path_mgd[:,0],theta_path_mgd[:,1],"g-+",linewidth=1,label='Mini-batch')
plt.plot(theta_path_bgd[:,0],theta_path_bgd[:,1],"b-o",linewidth=1,label='Batch')
plt.legend(loc="upper left",fontsize=16)
plt.xlabel(r"$\theta_0$",fontsize=20)
plt.ylabel(r"$\theta_1$",fontsize=20,rotation=0)
plt.axis([2.5,4.5,2.3,3.9])
save_fig("gradient_descent_paths_plot")
plt.show()