手写两层神经网络进行十分类(包含Dropout, BN, SGD momentum, weight decay, softmax,两层RuLU)USYD 5329 Assignmet1

1. Activition.py

from abc import ABC, abstractmethod

import numpy as np


class ActivationBase(ABC):
    def __init__(self, **kwargs):
        super().__init__()

    def __call__(self, z):
        if z.ndim == 1:
            z = z.reshape(1, -1)
        return self.forward(z)

    @abstractmethod
    def forward(self, z):
        """
        Forward propagation, obtain a through the activation function
        """
        raise NotImplementedError

    @abstractmethod
    def grad(self, x, **kwargs):
        """
        Get gradient from back
        """
        raise NotImplementedError


class ReLU(ActivationBase):
    """
    ReLU function unit
    """

    def __init__(self):
        super().__init__()

    def __str__(self):
        return "ReLU"

    def forward(self, z):
        return np.clip(z, 0, np.inf)

    def grad(self, x, **kwargs):
        return (x > 0).astype(int)




2. Objective.py

from abc import ABC, abstractmethod

import numpy as np


class ObjectiveBase(ABC):

    def __init__(self):
        super().__init__()

    @abstractmethod
    def loss(self, y_true, y_pred):
        """
        Calculate loss
        """
        raise NotImplementedError

    @abstractmethod
    def grad(self, y_true, y_pred, **kwargs):
        """
        Calculate the gradient of the cost function
        """
        raise NotImplementedError


class SquaredError(ObjectiveBase):
    """
    SEC function
    """

    def __init__(self):
        super().__init__()

    def __call__(self, y_true, y_pred):
        return self.loss(y_true, y_pred)

    def __str__(self):
        return "SquaredError"

    @staticmethod
    def loss(y_true, y_pred):
        """
        :param y_true:The true values of the n samples trained in the shape of a (n,m) array.
        :param y_pred:The predicted values of the n samples trained in the shape of a (n,m) array.
        """
        (n, _) = y_true.shape
        return 0.5 * np.linalg.norm(y_pred - y_true) ** 2 / n

    @staticmethod
    def grad(y_true, y_pred, z, acti_fn):
        (n, _) = y_true.shape
        return (y_pred - y_true) * acti_fn.grad(z) / n


class CrossEntropy(ObjectiveBase):
    """
    Cross-entropy cost function
    """

    def __init__(self):
        super().__init__()

    def __call__(self, y_true, y_pred):
        return self.loss(y_true, y_pred)

    def __str__(self):
        return "CrossEntropy"

    @staticmethod
    def loss(y_true, y_pred):
        """
        :param y_true:The true values of the n samples trained, which are required to be shaped as (n,m) binary (each sample is one-hot encoded).
        :param y_pred:The predicted values of the n samples trained in the shape of (n,m).
        """
        (n, _) = y_true.shape
        eps = np.finfo(float).eps  # Prevent np.log(0)
        cross_entropy = -np.sum(y_true * np.log(y_pred + eps)) / n
        return cross_entropy

    @staticmethod
    def grad(y_true, y_pred):
        (n, _) = y_true.shape
        grad = (y_pred - y_true) / n
        return grad

3. Optimizer.py

from abc import ABC, abstractmethod
import numpy as np


class OptimizerBase(ABC):
    def __init__(self):
        pass

    def __call__(self, params, params_grad, params_name):
        """
        :param params:parameters to be updated, such as the weight matrix W.
        :param params_grad:The gradient of the parameter to be updated.
        :param params_name:The name of the parameter to be updated.
        """
        return self.update(params, params_grad, params_name)

    @abstractmethod
    def update(self, params, params_grad, params_name):
        """
        :param params:parameters to be updated, such as the weight matrix W.
        :param params_grad:The gradient of the parameter to be updated.
        :param params_name:The name of the parameter to be updated.
        """
        raise NotImplementedError


class SGD(OptimizerBase):
    """
    sgd Optimization methods
    """

    def __init__(self, lr=0.01):
        super().__init__()
        self.lr = lr
        self.cache = {}

    def __str__(self):
        return "SGD(lr={})".format(self.hyperparams["lr"])

    def update(self, params, params_grad, params_name):
        update_value = self.lr * params_grad
        return params - update_value

    @property
    def hyperparams(self):
        return {"op": "SGD", "lr": self.lr
                }


"""
SGD momentum
"""


class Momentum(OptimizerBase):
    def __init__(
            self, lr=0.001, momentum=0.0, **kwargs
    ):
        """
        :param lr:Learning rate, float (default: 0.001)
        :param momentum:The alpha when considering Momentum, which determines how fast the previous gradient contribution decays, takes a value in the range [0, 1], default 0
        """
        super().__init__()
        self.lr = lr
        self.momentum = momentum
        self.cache = {}

    def __str__(self):
        return "Momentum(lr={}, momentum={})".format(self.lr, self.momentum)

    def update(self, param, param_grad, param_name):
        C = self.cache
        lr, momentum = self.lr, self.momentum
        if param_name not in C:  # save v
            C[param_name] = np.zeros_like(param_grad)
        update = momentum * C[param_name] - lr * param_grad
        self.cache[param_name] = update
        return param + update

    @property
    def hyperparams(self):
        return {"op": "Momentum", "lr": self.lr,
                "momentum": self.momentum
                }

4. Regularizer.py

from abc import ABC, abstractmethod

import numpy as np


class RegularBase(ABC):
    def __init__(self, **kwargs):
        super().__init__()

    @abstractmethod
    def loss(self, **kwargs):
        raise Not
  • 0
    点赞
  • 1
    收藏
    觉得还不错? 一键收藏
  • 0
    评论

“相关推荐”对你有帮助么?

  • 非常没帮助
  • 没帮助
  • 一般
  • 有帮助
  • 非常有帮助
提交
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值