手写两层神经网络进行十分类（包含Dropout， BN， SGD momentum， weight decay， softmax，两层RuLU）USYD 5329 Assignmet1

1. Activition.py

from abc import ABC, abstractmethod

import numpy as np


class ActivationBase(ABC):
    def __init__(self, **kwargs):
        super().__init__()

    def __call__(self, z):
        if z.ndim == 1:
            z = z.reshape(1, -1)
        return self.forward(z)

    @abstractmethod
    def forward(self, z):
        """
        Forward propagation, obtain a through the activation function
        """
        raise NotImplementedError

    @abstractmethod
    def grad(self, x, **kwargs):
        """
        Get gradient from back
        """
        raise NotImplementedError


class ReLU(ActivationBase):
    """
    ReLU function unit
    """

    def __init__(self):
        super().__init__()

    def __str__(self):
        return "ReLU"

    def forward(self, z):
        return np.clip(z, 0, np.inf)

    def grad(self, x, **kwargs):
        return (x > 0).astype(int)

2. Objective.py

from abc import ABC, abstractmethod

import numpy as np


class ObjectiveBase(ABC):

    def __init__(self):
        super().__init__()

    @abstractmethod
    def loss(self, y_true, y_pred):
        """
        Calculate loss
        """
        raise NotImplementedError

    @abstractmethod
    def grad(self, y_true, y_pred, **kwargs):
        """
        Calculate the gradient of the cost function
        """
        raise NotImplementedError


class SquaredError(ObjectiveBase):
    """
    SEC function
    """

    def __init__(self):
        super().__init__()

    def __call__(self, y_true, y_pred):
        return self.loss(y_true, y_pred)

    def __str__(self):
        return "SquaredError"

    @staticmethod
    def loss(y_true, y_pred):
        """
        :param y_true：The true values of the n samples trained in the shape of a (n,m) array.
        :param y_pred：The predicted values of the n samples trained in the shape of a (n,m) array.
        """
        (n, _) = y_true.shape
        return 0.5 * np.linalg.norm(y_pred - y_true) ** 2 / n

    @staticmethod
    def grad(y_true, y_pred, z, acti_fn):
        (n, _) = y_true.shape
        return (y_pred - y_true) * acti_fn.grad(z) / n


class CrossEntropy(ObjectiveBase):
    """
    Cross-entropy cost function
    """

    def __init__(self):
        super().__init__()

    def __call__(self, y_true, y_pred):
        return self.loss(y_true, y_pred)

    def __str__(self):
        return "CrossEntropy"

    @staticmethod
    def loss(y_true, y_pred):
        """
        :param y_true：The true values of the n samples trained, which are required to be shaped as (n,m) binary (each sample is one-hot encoded).
        :param y_pred：The predicted values of the n samples trained in the shape of (n,m).
        """
        (n, _) = y_true.shape
        eps = np.finfo(float).eps  # Prevent np.log(0)
        cross_entropy = -np.sum(y_true * np.log(y_pred + eps)) / n
        return cross_entropy

    @staticmethod
    def grad(y_true, y_pred):
        (n, _) = y_true.shape
        grad = (y_pred - y_true) / n
        return grad

3. Optimizer.py

from abc import ABC, abstractmethod
import numpy as np


class OptimizerBase(ABC):
    def __init__(self):
        pass

    def __call__(self, params, params_grad, params_name):
        """
        :param params：parameters to be updated, such as the weight matrix W.
        :param params_grad：The gradient of the parameter to be updated.
        :param params_name：The name of the parameter to be updated.
        """
        return self.update(params, params_grad, params_name)

    @abstractmethod
    def update(self, params, params_grad, params_name):
        """
        :param params：parameters to be updated, such as the weight matrix W.
        :param params_grad：The gradient of the parameter to be updated.
        :param params_name：The name of the parameter to be updated.
        """
        raise NotImplementedError


class SGD(OptimizerBase):
    """
    sgd Optimization methods
    """

    def __init__(self, lr=0.01):
        super().__init__()
        self.lr = lr
        self.cache = {}

    def __str__(self):
        return "SGD(lr={})".format(self.hyperparams["lr"])

    def update(self, params, params_grad, params_name):
        update_value = self.lr * params_grad
        return params - update_value

    @property
    def hyperparams(self):
        return {"op": "SGD", "lr": self.lr
                }


"""
SGD momentum
"""


class Momentum(OptimizerBase):
    def __init__(
            self, lr=0.001, momentum=0.0, **kwargs
    ):
        """
        :param lr：Learning rate, float (default: 0.001)
        :param momentum：The alpha when considering Momentum, which determines how fast the previous gradient contribution decays, takes a value in the range [0, 1], default 0
        """
        super().__init__()
        self.lr = lr
        self.momentum = momentum
        self.cache = {}

    def __str__(self):
        return "Momentum(lr={}, momentum={})".format(self.lr, self.momentum)

    def update(self, param, param_grad, param_name):
        C = self.cache
        lr, momentum = self.lr, self.momentum
        if param_name not in C:  # save v
            C[param_name] = np.zeros_like(param_grad)
        update = momentum * C[param_name] - lr * param_grad
        self.cache[param_name] = update
        return param + update

    @property
    def hyperparams(self):
        return {"op": "Momentum", "lr": self.lr,
                "momentum": self.momentum
                }

4. Regularizer.py

from abc import ABC, abstractmethod

import numpy as np


class RegularBase(ABC):
    def __init__(self, **kwargs):
        super().__init__()

    @abstractmethod
    def loss(self, **kwargs):
        raise Not