神经网络常用激活函数-正反传播代码实现

最近在学习神经网络，在此我总结了全连接神经网络常用的激活函数的正反传播，如有误请提出，一起学习

Sigmoid

class Sigmoid:

    def forward(self,x):
        out = 1 / (1 + np.exp(-x))
        self.out = out

        return out
    
    def backward(self,dout):
        dx = dout * (1.0 - self.out) * self.out

        return dx

ReLu

class ReLu:

    def forward(self,x):
        self.mask = (x <= 0)
        out = x.copy()
        out[self.mask] = 0

        return out
    
    def backward(self,dout):
        dout[self.mask] = 0
        dx = dout

        return dx

Affine

class Affine:

    def forward(self,x,w,b):
        self.x = x
        self.w = w
        self.b = b
        out = np.dot(self.w,x) + self.b

        return out
    
    def backward(self,dout):
        dx = np.dot(dout,self.w.T)
        dw = np.dot(dout,self.x.T)
        db = np.sum(dout,axis = 0)
        
        return dx

SoftmaxWithLoss

class SoftmaxWithLoss:

    def __init__(self):
        self.loss = None
        self.y = None  #输出数据
        self.t = None  #监督数据（one-hot）
        
    def forward(self,x,t):
        self.t = t
        self.y = Softmax(x)
        self.loss = cross_entropy_error(self.y,self.t)
        
        return self.loss
    
    def backward(self,dout = 1):
        batch_size = self.t.shape[0]
        dx = (self.y - self.t) / batch_size

        return dx

    #softmax函数
    def Softmax(x):
        c = np.max(x)
        exp_x = np.exp(x-c)#减去最大值防止指数运算溢出
        sum_exp_x = np.sum(np.exp(x-c))
        y = exp_x/sum_exp_x
        
        return y
        
    #cross_entropy_error函数
    def cross_entropy_error(y,t):
        delta = 1e-7
        return -np.sum(t * np.log(y+delta))

代码来自《深度学习入门-基于Python的理论与实现》