逻辑回归（公式推导+numpy实现）

西红柿爱喝水

于 2022-07-16 10:42:38 发布

阅读量182

点赞数

分类专栏：有监督学习文章标签：逻辑回归 numpy 机器学习 python

本文链接：https://blog.csdn.net/qq_42800926/article/details/125816632

版权

有监督学习专栏收录该内容

2 篇文章 0 订阅

订阅专栏

公式推导

含义	公式	维度
输入（矩阵形式）	$\mathbf X= \begin{bmatrix}-\mathbf {x^{(1)}}^T - \\-\mathbf {x^{(2)}}^T- \\\cdots\\-\mathbf {x^{(i)}}^T-\\\cdots\\-\mathbf {x^{(m)}}^T-\end{bmatrix}$	$m\times n$
输入	$\mathbf x^{(i)}=\begin{bmatrix} x_{1}^{(i)} & x_{2}^{(i)} & \cdots & x_{j}^{(i)} & \cdots & x_{n}^{(i)}\end{bmatrix}^T$	$n\times 1$
标签	$\mathbf y={\begin{bmatrix} y^{(1)} & y^{(2)} & \cdots & y^{(i)} &\cdots &y^{(m)}\end{bmatrix}}^T$	$m\times 1$
参数	$\mathbf w={\begin{bmatrix}w_{1} & w_{2} & \cdots & w_{j} & \cdots & w_{n}\end{bmatrix}}^T$	$n\times 1$
输出	$\begin{aligned}f_{\mathbf w,b}(\mathbf x^{(i)}) &=g({\mathbf w}^T{\mathbf x}^{(i)} + b) \\ g(z) &= \frac{1}{1+e^{-z}} \end{aligned}$	标量
输出（矩阵形式）	$f_{\mathbf w,b}(\mathbf X) = g(\mathbf X \mathbf w+ b)$	$m\times 1$
预测	$\hat{y}^{(i)}= \begin{cases} 1 & \text{if }f_{\mathbf w,b}(\mathbf x^{(i)})\ge 0.5\\ 0 & \text{if }f_{\mathbf w,b}(\mathbf x^{(i)}) <0.5\end{cases}$	标量
损失函数	$\begin{aligned}cost^{(i)} &= \begin{cases} -\log\left(f_{\mathbf{w},b}\left( \mathbf{x}^{(i)} \right) \right) & \text{if }y^{(i)}=1\\ -\log \left( 1 - f_{\mathbf{w},b}\left( \mathbf{x}^{(i)} \right) \right)&\text{if }y^{(i)}=0\end{cases} \\ &=-y^{(i)} \log\left(f_{\mathbf{w},b}\left( \mathbf{x}^{(i)} \right) \right) - \left( 1 - y^{(i)}\right) \log \left( 1 - f_{\mathbf{w},b}\left( \mathbf{x}^{(i)} \right) \right)\end{aligned}$	标量
代价函数	$\begin{aligned}J(\mathbf w,b) &= \frac{1}{m} \sum\limits_{i = 1}^{m} cost^{(i)}+\frac{\lambda}{2m}\sum\limits_{j = 1}^{n} w_{j}^2\\&=\frac{1}{m}\left(-\mathbf y^Tlog(f_{\mathbf w,b}(\mathbf X))-(\Iota-\mathbf y)^Tlog(\Iota-f_{\mathbf w,b}(\mathbf X))\right)+\frac{\lambda}{2m}\mathbf w^T\mathbf w\end{aligned}$	标量
梯度下降	$\begin{aligned}w_j :&= w_j - \alpha \frac{\partial J(\mathbf{w},b)}{\partial w_j}\\ b :&= b - \alpha \frac{\partial J(\mathbf{w},b)}{\partial b}\\\frac{\partial J(\mathbf{w},b)}{\partial w_j} &= \frac{1}{m} \sum\limits_{i = 1}^{m} (f_{\mathbf{w},b}(\mathbf{x}^{(i)}) - y^{(i)})x_{j}^{(i)} + \frac{\lambda}{m} w_j \\ \frac{\partial J(\mathbf{w},b)}{\partial b} &= \frac{1}{m} \sum\limits_{i = 1}^{m} (f_{\mathbf{w},b}(\mathbf{x}^{(i)}) - y^{(i)}) \end{aligned}$	标量
梯度下降（矩阵形式）	$\begin{aligned}\mathbf w:&=\mathbf w-\alpha\frac{\partial J(\mathbf{w},b)}{\partial \mathbf{w}}\\ \frac{\partial J(\mathbf{w},b)}{\partial \mathbf{w}}&=\frac{1}{m}\mathbf X^T(f_{\mathbf w,b}(\mathbf X) -\mathbf y)+\frac{\lambda}{m} \mathbf w\end{aligned}$	$n\times 1$

numpy实现

def zscore_normalize_features(X):
    mu=np.mean(X,axis=0)                 
    sigma=np.std(X,axis=0)           
    X_norm=(X-mu)/sigma      
    return X_norm,mu,sigma
# SIGMOID
def sig(z):
 
    return 1/(1+np.exp(-z))
# f_wb
def compute_f_wb(X,w,b):
    f_wb=sig(np.dot(X,w)+b)# (m,1)
    return f_wb
# j_wb
def compute_cost(X,y,w,b,lambda_,f_wb_function):
    m,n=X.shape
    f_wb=f_wb_function(X,w,b) # m*1
    j_wb=1/m*(-np.dot(y.T, np.log(f_wb))-np.dot((1-y).T,np.log(1-f_wb)))+(lambda_/2*m)*np.dot(w.T,w)# (1,1)
    j_wb=j_wb[0,0] # scalar  
    return j_wb
# dj_dw,dj_db
def compute_gradient(X, y, w, b, lambda_,f_wb_function): 
    m,n=X.shape
    f_wb=f_wb_function(X,w,b) # m*1
    
    dj_dw=(1/m)*np.dot(X.T,(f_wb-y))+(lambda_/n)*w # n*1
    dj_db=(1/m)*np.sum(f_wb-y) # scalar
    return dj_dw,dj_db
# w,b,j_history,w_history
def gradient_descent(X, y, w, b, cost_function, gradient_function, f_wb_function,alpha, num_iters,lambda_): 
    J_history = []
    w_history = []
    w_temp = copy.deepcopy(w)  
    b_temp = b
    for i in range(num_iters):
        dj_dw, dj_db = gradient_function(X, y, w_temp,b_temp,lambda_,f_wb_function)    
        w_temp = w_temp  - alpha * dj_dw               
        b_temp  = b_temp  - alpha * dj_db      
        cost =  cost_function(X, y, w_temp, b_temp,lambda_,f_wb_function)
        J_history.append(cost)  
    return w_temp, b_temp, J_history, w_history

样本点

x_train, y_train = load_data("data/ex2data2.txt")
y_train=y_train.reshape(-1,1)
x_train.shape,y_train.shape
fig=go.Figure()
fig.add_trace(
    go.Scatter(
        x=x_train[np.where(y_train==0)[0],0],
        y=x_train[np.where(y_train==0)[0],1],
        mode="markers",
        name="第一类"
    )
)
fig.add_trace(
    go.Scatter(
        x=x_train[np.where(y_train==1)[0],0],
        y=x_train[np.where(y_train==1)[0],1],
        mode="markers",
        name="第一类"
    )
)
fig.update_layout(width=1000,height=618)
fig.show()

在这里插入图片描述

特征

def map_feature(X1, X2,degree):
    """
    Feature mapping function to polynomial features    
    """
    X1 = np.atleast_1d(X1)
    X2 = np.atleast_1d(X2)
    out = []
    for i in range(1, degree+1):
        for j in range(i + 1):
            out.append((X1**(i-j) * (X2**j)))
    return np.stack(out, axis=1)
    
feature_power=3
features=map_feature(x_train[:, 0], x_train[:, 1],feature_power)
features.shape
x_,mu,sigma=zscore_normalize_features(features)
y_=y_train
x_.shape,y_.shape

梯度下降

m,n=x_.shape
initial_w = np.zeros((n,1))
initial_b = 0
iterations = 1500
alpha = 0.3
lambda_=0
w,b,J_history,w_history = gradient_descent(x_ ,y_, initial_w, initial_b, compute_cost, compute_gradient, compute_f_wb,alpha, iterations,lambda_)

fig=go.Figure()
fig.update_layout(width=1000,height=618)
fig.add_trace(
    go.Scatter(
        x=np.arange(1,iterations+1),
        y=J_history,
        name="学习曲线",
        mode="markers+lines"
    )
)
fig.update_layout(
    xaxis_title="迭代次数",
    yaxis_title="J_wb"
)
fig.show()

在这里插入图片描述

决策边界

u=np.linspace(-1.5,1.5,100)
v=np.linspace(-1.5,1.5,100)
z = np.zeros((len(u), len(v)))

for i in range(len(u)):
    for j in range(len(v)):
        temp=(map_feature(u[i], v[j],feature_power).T-mu.reshape(-1,1))/sigma.reshape(-1,1)
        z[i,j] = sig(np.dot(w.T,temp) + b)[0,0]
        
fig=go.Figure()
fig.update_layout(width=1000,height=618)
fig.add_trace(
    go.Contour(
        z=z,
        contours_coloring='lines',
        x=u,
        y=v,
    )
)
fig.add_trace(
    go.Scatter(
        x=x_train[np.where(y_train==0)[0],0],
        y=x_train[np.where(y_train==0)[0],1],
        mode="markers",
        name="第一类"
    )
)
fig.add_trace(
    go.Scatter(
        x=x_train[np.where(y_train==1)[0],0],
        y=x_train[np.where(y_train==1)[0],1],
        mode="markers",
        name="第一类"
    )
)
fig.show()