逻辑回归

S_ssssssk

于 2020-03-08 14:02:41 发布

阅读量121

点赞数

文章标签：机器学习 python 逻辑回归

本文链接：https://blog.csdn.net/weixin_43859153/article/details/104683882

版权

逻辑回归

目标函数

一个模型是线性还是非线性是这个模型的决策边界所决定的，因此逻辑是线性的分类器。
$\begin{array}{c}P\left(y_{i}=1 | x_{i}\right)=\frac{1}{1+e^{-\beta^{T} x_{i}}} \\ P\left(y_{i}=0 | x_{i}\right)=1-\frac{1}{1+e^{-\beta^{T} x_{i}}}\end{array}$
用交叉熵损失函数来表示
$\begin{aligned} f(\beta) &=-\sum_{i=1}^{n}\left[y_{i} \ln p\left(y_{i}=1 | x_{i}\right)+\left(1-y_{i}\right) \ln p\left(y_{i}=0 | x_{i}\right)\right] \\ &=-\sum_{i=1}^{n}\left[y_{i} \ln \left(\frac{1}{1+e^{-\beta^{\mathrm{T}} x_{i}}}\right)+\left(1-y_{i}\right) \ln \left(\frac{e^{-\beta^{\mathrm{T}} x_{i}}}{1+e^{-\beta^{\mathrm{T}} x_{i}}}\right)\right] \\ &=\sum_{i=1}^{n}\left[\ln \left(1+e^{\beta^{T} x_{i}}\right)-y_{i} \beta^{\mathrm{T}} x_{i}\right] \end{aligned}$

梯度下降法

$\begin{array}{l}\frac{\partial f(\beta)}{\partial \beta}=-\sum_{i=1}^{n} x_{i}\left(y_{i}-P\left(y_{i}=1 | x_{i}\right)\right) \\ \frac{\partial^{2} f(\beta)}{\partial \beta \partial \beta^{T}}=\sum_{i=1}^{n} x_{i} x_{i}^{T} P\left(y_{i}=1 | x_{i}\right) P\left(y_{i}=0 | x_{i}\right)\end{array}$

更新公式：
$\beta^{t+1}=\beta^{t}-\alpha\displaystyle\frac{\partial f(\beta)}{\partial \beta}$

梯度下降法实现

批量梯度下降法(Batch Gradient Descent)

    def BatchGradDesc(self):
        """批量梯度下降法(Batch Gradient Descent)"""
        m,n = self.X.shape
        b = np.ones((m,1))
        X = np.concatenate([b,self.X],axis=1) #在X前加入一列1，当作b
        X = np.mat(X)
        y = np.mat(self.y).transpose()
        weights = np.zeros((n+1,1)) #初始化权重
        for i in range(self.max_iter): #梯度下降迭代
            p = self.sigmoid(X * weights)                           
            error = p - y
            weights = weights - self.lr * X.transpose() * error
        return weights.getA()

随机梯度下降法(Stochastic Gradient Descent)

    def StocGradDesc(self):
        """随机梯度下降法(Stochastic Gradient Descent)"""
        b = np.ones((self.X.shape[0],1))
        data = np.concatenate([b,self.X,self.y[:,np.newaxis]],axis=1) #在X前加入一列1，当作b
        data = pd.DataFrame(data).sample(self.max_iter, replace=True)
        data.index = range(data.shape[0])
        X = np.mat(data.iloc[:,:-1].values)
        y = np.mat(data.iloc[:,-1].values).transpose()
        m,n = X.shape
        weights = np.zeros((n,1)) #初始化权重
        for i in range(m):
            p = self.sigmoid(X[i,:] * weights) 
            error = p - y[i]
            weights = weights - self.lr * X[i,:].transpose() * error
        return weights.getA()

小批量梯度下降法(Mini-Batch Gradient Descent)

    def MiniBatchBGradDesc(self):
        """小批量梯度下降法(Mini-Batch Gradient Descent)"""
        batch = 20
        b = np.ones((self.X.shape[0],1))
        data = np.concatenate([b,self.X,self.y[:,np.newaxis]],axis=1) #在X前加入一列1，当作b
        data = pd.DataFrame(data).sample(batch*self.max_iter, replace=True)
        data.index = range(data.shape[0])
        X = np.mat(data.iloc[:,:-1].values)
        y = np.mat(data.iloc[:,-1].values).transpose()
        m,n = X.shape
        weights = np.zeros((n,1)) #初始化权重
        index1 = 0
        index2 = batch
        for i in range(m):
            p = self.sigmoid(X[index1:index2,:] * weights) 
            error = p - y[index1:index2]
            weights = weights - self.lr * X[index1:index2,:].transpose() * error
            index1 += batch
            index2 += batch
        return weights.getA()

完整代码

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.datasets import make_blobs
from sklearn.datasets import load_breast_cancer
from sklearn.preprocessing import StandardScaler

#二维数据测试
#X, y = make_blobs(n_samples=20, centers=2, n_features=2, cluster_std=1, random_state=0)

breast = load_breast_cancer()
X = breast.data
y = breast.target
scaler = StandardScaler()
X = scaler.fit_transform(X)


class Logistic(object):
    def __init__(self, X, y, lr=0.01, max_iter=1000, solver='BatchGradDesc'):
        self.X = X
        self.y = y
        self.lr = lr
        self.max_iter = max_iter
        self.solver = solver
    
    def sigmoid(self, x):
        """sigmoid函数"""
        return 1.0 / (1 + np.exp(-x))

    def pre(self, weights):
        """预测函数"""
        m,n = self.X.shape
        b = np.ones((m,1))
        X = np.concatenate([b,self.X],axis=1)
        prob = self.sigmoid(np.dot(X,weights))
        pre = np.where(prob>0.5, 1, 0)
        return pre

    def Accuracy(self, pre):
        """计算正确率"""
        m = self.y.shape[0]
        y = self.y.reshape(m,1)
        n_accuracy = (pre == y).sum()
        accuracy = round(n_accuracy *100 / m ,2)
        return accuracy

    def BatchGradDesc(self):
        """批量梯度下降法(Batch Gradient Descent)"""
        m,n = self.X.shape
        b = np.ones((m,1))
        X = np.concatenate([b,self.X],axis=1) #在X前加入一列1，当作b
        X = np.mat(X)
        y = np.mat(self.y).transpose()
        weights = np.zeros((n+1,1)) #初始化权重
        for i in range(self.max_iter): #梯度下降迭代
            p = self.sigmoid(X * weights)                           
            error = p - y
            weights = weights - self.lr * X.transpose() * error
        return weights.getA()                                         
    
    def StocGradDesc(self):
        """随机梯度下降法(Stochastic Gradient Descent)"""
        b = np.ones((self.X.shape[0],1))
        data = np.concatenate([b,self.X,self.y[:,np.newaxis]],axis=1) #在X前加入一列1，当作b
        data = pd.DataFrame(data).sample(self.max_iter, replace=True)
        data.index = range(data.shape[0])
        X = np.mat(data.iloc[:,:-1].values)
        y = np.mat(data.iloc[:,-1].values).transpose()
        m,n = X.shape
        weights = np.zeros((n,1)) #初始化权重
        for i in range(m):
            p = self.sigmoid(X[i,:] * weights) 
            error = p - y[i]
            weights = weights - self.lr * X[i,:].transpose() * error
        return weights.getA()  

    def MiniBatchBGradDesc(self):
        """小批量梯度下降法(Mini-Batch Gradient Descent)"""
        batch = 20
        b = np.ones((self.X.shape[0],1))
        data = np.concatenate([b,self.X,self.y[:,np.newaxis]],axis=1) #在X前加入一列1，当作b
        data = pd.DataFrame(data).sample(batch*self.max_iter, replace=True)
        data.index = range(data.shape[0])
        X = np.mat(data.iloc[:,:-1].values)
        y = np.mat(data.iloc[:,-1].values).transpose()
        m,n = X.shape
        weights = np.zeros((n,1)) #初始化权重
        index1 = 0
        index2 = batch
        for i in range(m):
            p = self.sigmoid(X[index1:index2,:] * weights) 
            error = p - y[index1:index2]
            weights = weights - self.lr * X[index1:index2,:].transpose() * error
            index1 += batch
            index2 += batch
        return weights.getA()  

#    def plot(self, weights):
#        """二维数据画图"""
#        plt.scatter(self.X[self.y==1,0],self.X[self.y==1,1])
#        plt.scatter(self.X[self.y==0,0],self.X[self.y==0,1])
#        plt.plot(self.X[:,0], (-self.X[:,0]*weights[1,0]-weights[0,0])/weights[2,0])

    def run(self):
        if self.solver == 'BatchGradDesc':
            weights = self.BatchGradDesc()
        elif self.solver == 'StocGradDesc':
            weights = self.StocGradDesc()
        else:
            weights = self.MiniBatchBGradDesc()
        #self.plot(weights)
        pre = self.pre(weights)
        accuracy = self.Accuracy(pre)
        print("{}正确率为：".format(self.solver),accuracy)
        return weights,pre
    
weights1,pre1=Logistic(X, y, solver='BatchGradDesc').run()
weights2,pre2=Logistic(X, y, solver='StocGradDesc').run()
weights3,pre3=Logistic(X, y, solver='MiniBatchBGradDesc').run()

输出

BatchGradDesc正确率为： 98.95
StocGradDesc正确率为： 98.07
MiniBatchBGradDesc正确率为： 98.59

梯度下降法优缺点

   1）批量梯度下降法BGD(Batch Gradient Descent):

         针对的是整个数据集，通过对所有的样本的计算来求解梯度的方向。

         优点：全局最优解；易于并行实现；

         缺点：当样本数据很多时，计算量开销大，计算速度慢

    2）小批量梯度下降法MBGD（mini-batch Gradient Descent）

        把数据分为若干个批，按批来更新参数，这样，一个批中的一组数据共同决定了本次梯度的方向，下降起来就不容易跑偏，减少了随机性

        优点：减少了计算的开销量，降低了随机性

     3）随机梯度下降法SGD（stochastic gradient descent）

        每个数据都计算算一下损失函数，然后求梯度更新参数。

        优点：计算速度快

        缺点：收敛性能不好

    总结：SGD可以看作是MBGD的一个特例，及batch_size=1的情况。

S_ssssssk

关注

0
点赞
踩
1

收藏

觉得还不错? 一键收藏
0
评论
逻辑回归

逻辑回归目标函数一个模型是线性还是非线性是这个模型的决策边界所决定的，因此逻辑是线性的分类器。P(yi=1∣xi)=11+e−βTxiP(yi=0∣xi)=1−11+e−βTxi\begin{array}{c}P\left(y_{i}=1 | x_{i}\right)=\frac{1}{1+e^{-\beta^{T} x_{i}}} \\ P\left(y_{i}=0 | x_{i}\ri...
复制链接

扫一扫