logistic算法实现（gd,sgd,newton）(iris，mnist应用)

1.logistic实现

import numpy as np
import matplotlib.pyplot as plt

class Logistic():
    def __init__(self):
       pass

    #GD
    def fit_by_gd(self, X, y, x_test, y_test, lr=0.1, N=1000, eval=False, eval_step=50):
        m,n=X.shape
        w=np.zeros((n,1))
        self.w = w
        loss=[]
        for t in range(1,N+1):
            h=self._sigmod(X.dot(w))
            g=1.0/m*X.T.dot(h-y)
            w=w-lr*g
            if eval and t%eval_step==0:
                loss.append(self.eval_by_crossEntropy(y_test,self.predict_percentage(x_test)))
            self.w=w
        return w,loss

    #stochastic gradient descent，SGD
    def fit_by_sgd(self,X,y,x_test,y_test,eta_0=10,eta_1=50,N=1000,eval=False,eval_step=50):
        m,n=X.shape
        w=np.zeros((n,1))
        self.w=w
        loss=[]
        for t in range(1,N+1):
            i=np.random.randint(m)
            x=X[i].reshape(1,-1)
            pred=self._sigmod(x.dot(w))
            g=x.T*(pred-y[i])
            w=w-eta_0/(t+eta_1)*g
            self.w+=w
            if eval and t%eval_step==0:
                y_pred=self._sigmod(x_test.dot(self.w/t))
                loss.append(np.average(-y_test*np.log(y_pred)-(1-y_test)*np.log(1-y_pred)))
        self.w/=N
        return w,loss

    #Newton's method
    def fit_by_Newton(self,X, y, x_test, y_test, N=1000, eval=False, eval_step=50):
        m,n=X.shape
        w=np.zeros((n,1))
        loss=[]
        for t in range(1,N+1):
            pred=self._sigmod(X.dot(w))
            g=1.0/m*X.T.dot(pred-y)
            pred=pred.reshape(-1)
            D=np.diag(pred*(1-pred))
            H=1.0/m*(X.T.dot(D)).dot(X)
            try:
                w-=np.linalg.inv(H).dot(g)
            except:
                print(str(t),":Matrix irreversibility")
            if eval and t%eval_step==0:
                y_pred=self._sigmod(x_test.dot(w))
                loss.append(np.average(-y_test*np.log(y_pred)-(1-y_test)*np.log(1-y_pred)))
        self.w=w
        return w,loss

    def predict_percentage(self,X):
        return self._sigmod(X.dot(self.w))

    def predict_thresh(self,X,thresh=0.5):
        prob=self.predict_percentage(X)
        return (prob>thresh).astype(np.int)

    def eval_by_crossEntropy(self,y_true,y_pred):
        return np.average(-y_true*np.log(y_pred)-(1-y_true)*np.log(1-y_pred))

    def _sigmod(self,scores):
        return  1 / (1 + np.exp(-scores))

    def show_loss(self,loss,title="loss",epoch_step=50):
        x=np.linspace(1,len(loss)+1,len(loss))*epoch_step
        plt.title(title)
        plt.plot(x,loss,label="loss")
        plt.xlabel("epoch")
        plt.ylabel("loss")
        plt.legend()
        plt.show()

    def show_losses(self,loss,epoch_step=50):
        plt.title("loss")
        for key,value in loss.items():
            x = np.linspace(1, len(value) + 1, len(value)) * epoch_step
            plt.plot(x,value,label=key)
        plt.xlabel("epoch")
        plt.ylabel("crossEntropy")
        plt.legend()
        plt.show()

2.山鸢尾识别

import numpy as np
from sklearn import datasets
from sklearn.model_selection import train_test_split
import logistic

def load_data():
    #process dataset
    iris=datasets.load_iris()
    x=iris["data"]
    y=(iris["target"]==2).astype(np.int).reshape(-1,1)
    x_train,x_test,y_train,y_test=train_test_split(x,y,test_size=0.2,random_state=0)

    x_train=np.c_[np.ones((x_train.shape[0],1)),x_train]
    x_test=np.c_[np.ones((x_test.shape[0],1)),x_test]

    return x_train, y_train, x_test, y_test

def iris(method="gd",eval_step=50):
    x_train, y_train, x_test, y_test = load_data()

    model = logistic.Logistic()
    print(method+":")
    if method=="gd":
        # use gd to fit and eval
        w, loss = model.fit_by_gd(x_train, y_train, x_test, y_test, lr=0.1, N=50000, eval=True, eval_step=eval_step)
    if method=="sgd":
        #use sgd to fit model
        w, loss = model.fit_by_sgd(x_train, y_train, x_test, y_test, N=50000, eval=True, eval_step=eval_step)
    if method == "newton":
        w, loss = model.fit_by_Newton(x_train, y_train, x_test, y_test, N=50,eval=True, eval_step=eval_step)

    y_pred_percentage=model.predict_percentage(x_test)
    # y_pred_thresh=model.predict_thresh(x_test,thresh=0.5)
    eval_percentage=model.eval_by_crossEntropy(y_test,y_pred_percentage)
    print("crossentropy",eval_percentage)

    model.show_loss(loss,title=method+" iris",epoch_step=eval_step)

def iris_all_methods():
    x_train, y_train, x_test, y_test = load_data()

    model = logistic.Logistic()
    # use gd to fit and eval
    _, loss1 = model.fit_by_gd(x_train, y_train, x_test, y_test, lr=0.1, N=50000, eval=True, eval_step=50)
    # use sgd to fit model
    _, loss2 = model.fit_by_sgd(x_train, y_train, x_test, y_test, N=50000, eval=True, eval_step=50)
    # use newton to fit and eval
    _, loss3 = model.fit_by_Newton(x_train, y_train, x_test, y_test, N=50000, eval=True, eval_step=50)

    loss = {"gd": loss1, "sgd":loss2,"newton":loss3}
    model.show_losses(loss,epoch_step=50)


if __name__=="__main__":
    iris(method="gd")
    iris(method="sgd")
    iris(method="newton",eval_step=1)

    iris_all_methods()

3.手写数字6识别

import numpy as np
import torchvision.datasets as dsets
import torchvision.transforms as transforms
from torch.utils.data import DataLoader
import logistic

def load_data():
    # load data
    train_dataset = dsets.MNIST(root="./mnist_data",
                                train=True,
                                transform=transforms.ToTensor(),
                                download=True)
    test_dataset = dsets.MNIST(root="./mnist_data",
                               train=False,
                               transform=transforms.ToTensor())

    train_data=DataLoader(train_dataset,batch_size=len(train_dataset),shuffle=True,num_workers=3)
    test_data = DataLoader(test_dataset, batch_size=len(test_dataset), shuffle=False,num_workers=3)
    for  x,y in train_data:
        x_train, y_train =np.array(x).squeeze(1),np.array(y)
        x_train=x_train.reshape(len(x_train),-1)
        y_train=(y_train==6).astype(np.int).reshape(-1,1)
    for x,y in test_data:
        x_test, y_test = np.array(x).squeeze(1), np.array(y)
        x_test=x_test.reshape(len(x_test),-1)
        y_test=(y_test==6).astype(np.int).reshape(-1,1)

    return x_train,y_train,x_test,y_test

def mnist(method="sgd",N=20000,eval=True,eval_step=50):
    x_train, y_train, x_test, y_test=load_data()

    model = logistic.Logistic()
    print(method+":")
    if method=="gd":
        # use gd to fit and eval
        w, loss = model.fit_by_gd(x_train[:1000], y_train[:1000], x_test[:300], y_test[:300], lr=0.1, N=N, eval=eval, eval_step=eval_step)
    if method=="sgd":
        # use sgd to fit and eval
        w,loss=model.fit_by_sgd(x_train,y_train,x_test,y_test,N=N,eval=eval,eval_step=eval_step)

    y_pred_percentage=model.predict_percentage(x_test)
    # y_pred_thresh=model.predict_thresh(x_test,thresh=0.5)
    eval_percentage=model.eval_by_crossEntropy(y_test,y_pred_percentage)
    print("crossentropy:",eval_percentage)
    model.show_loss(loss,title=method+" mnist 6",epoch_step=50)

def mnist_all_methods():
    x_train, y_train, x_test, y_test = load_data()

    model = logistic.Logistic()
    # use gd to fit and eval
    _, loss1 = model.fit_by_gd(x_train, y_train, x_test, y_test, lr=0.1, N=50000, eval=True, eval_step=50)
    # use sgd to fit model
    _, loss2 = model.fit_by_sgd(x_train, y_train, x_test, y_test, N=50000, eval=True, eval_step=50)
    # # use newton to fit and eval,but dimension too high,matrix irreversibility
    # _, loss3 = model.fit_by_Newton(x_train, y_train, x_test, y_test, N=50000, eval=True, eval_step=50)

    loss = {"gd": loss1, "sgd":loss2}
    model.show_losses(loss,epoch_step=50)


if __name__=="__main__":
    # mnist(method="gd")
    mnist(method="sgd")
    #
    # mnist_all_methods()