Deep Leaning 学习笔记之改善神经网络的超参数（1.4）—— 正则、dropout、梯度检验实例

1 正则化

1.1 L2正则化函数

def compute_cost_with_regularization(A3, Y, parameters, lambd):
    m = Y.shape[1]
    W1 = parameters["W1"]
    W2 = parameters["W2"]
    W3 = parameters["W3"]
    
    cross_entropy_cost = compute_cost(A3, Y)
    L2_regularization_cost=lambd/(2*m)*(np.sum(np.square(W1))+np.sum(np.square(W2))+np.sum(np.square(W3)))
    cost = cross_entropy_cost + L2_regularization_cost
    return cost

1.2 梯度下降

def backward_propagation_with_regularization(X, Y, cache, lambd):
    m = X.shape[1]
    (Z1, A1, W1, b1, Z2, A2, W2, b2, Z3, A3, W3, b3) = cache
    
    dZ3 = A3 - Y
    
    dW3 = 1./m * np.dot(dZ3, A2.T) + lambd/m*W3
    db3 = 1./m * np.sum(dZ3, axis=1, keepdims = True)
    
    dA2 = np.dot(W3.T, dZ3)
    dZ2 = np.multiply(dA2, np.int64(A2 > 0))
    dW2 = 1./m * np.dot(dZ2, A1.T) + lambd/m*W2
    db2 = 1./m * np.sum(dZ2, axis=1, keepdims = True)
    
    dA1 = np.dot(W2.T, dZ2)
    dZ1 = np.multiply(dA1, np.int64(A1 > 0))
    dW1 = 1./m * np.dot(dZ1, X.T) + lambd/m*W1
    db1 = 1./m * np.sum(dZ1, axis=1, keepdims = True)
    
    gradients = {"dZ3": dZ3, "dW3": dW3, "db3": db3,"dA2": dA2,
                 "dZ2": dZ2, "dW2": dW2, "db2": db2, "dA1": dA1, 
                 "dZ1": dZ1, "dW1": dW1, "db1": db1}
    
    return gradients

2 dropout

2.1 方程式

d3 = np.random.rand( a3.shape[0] , a3.shape[1] ) < keep_prop
（生成一个随机矩阵，当d3的元素值小于keep_prop时，为1，否则为0的 0-1矩阵）
a3 = np.multiply( a3 , d3 ) 也就是点乘， a3 $\ast$$=$ d3
a3 = a3 / keep_prop

2.2 函数（forword + backword propagation）

2.2.1 forword propagation with dropout

def forward_propagation_with_dropout(X, parameters, keep_prob = 0.5):
    np.random.seed(1)
    
    # retrieve parameters
    W1 = parameters["W1"]
    b1 = parameters["b1"]
    W2 = parameters["W2"]
    b2 = parameters["b2"]
    W3 = parameters["W3"]
    b3 = parameters["b3"]
    
    # LINEAR -> RELU -> LINEAR -> RELU -> LINEAR -> SIGMOID
    Z1 = np.dot(W1, X) + b1
    A1 = relu(Z1)
### Step 1: initialize matrix D1 = np.random.rand(..., ...)
    D1 = np.random.rand(A1.shape[0],A1.shape[1])      
### Step 2: convert entries of D1 to 0 or 1 (using keep_prob as the threshold)
    D1 = D1<keep_prob                                
### Step 3: shut down some neurons of A1
    A1 = np.multiply(A1,D1)                                        
### Step 4: scale the value of neurons that haven't been shut down
    A1 = A1/keep_prob                                 
    Z2 = np.dot(W2, A1) + b2
    A2 = relu(Z2)
### Step 1: initialize matrix D2 = np.random.rand(..., ...)
    D2 = np.random.rand(A2.shape[0],A2.shape[1]) 
### Step 2: convert entries of D2 to 0 or 1 (using keep_prob as the threshold)
    D2 = D2<keep_prob                                         
### Step 3: shut down some neurons of A2
    A2 = np.multiply(A2,D2)
### Step 4: scale the value of neurons that haven't been shut down
    A2 = A2/keep_prob                                         
    Z3 = np.dot(W3, A2) + b3
    A3 = sigmoid(Z3)
    
    cache = (Z1, D1, A1, W1, b1, Z2, D2, A2, W2, b2, Z3, A3, W3, b3)
    
    return A3, cache

2.2.2 backword propagation with dropout

def backward_propagation_with_dropout(X, Y, cache, keep_prob):
    m = X.shape[1]
    (Z1, D1, A1, W1, b1, Z2, D2, A2, W2, b2, Z3, A3, W3, b3) = cache
    
    dZ3 = A3 - Y
    dW3 = 1./m * np.dot(dZ3, A2.T)
    db3 = 1./m * np.sum(dZ3, axis=1, keepdims = True)
    dA2 = np.dot(W3.T, dZ3)
### Step 1: Apply mask D2 to shut down the same neurons as during the forward propagation
    dA2 = dA2*D2            
### Step 2: Scale the value of neurons that haven't been shut down
    dA2 = dA2/keep_prob              
    dZ2 = np.multiply(dA2, np.int64(A2 > 0))
    dW2 = 1./m * np.dot(dZ2, A1.T)
    db2 = 1./m * np.sum(dZ2, axis=1, keepdims = True)
    
    dA1 = np.dot(W2.T, dZ2)
### Step 1: Apply mask D1 to shut down the same neurons as during the forward propagation
    dA1 = dA1*D1              
### Step 2: Scale the value of neurons that haven't been shut down
    dA1 = dA1/keep_prob              
    dZ1 = np.multiply(dA1, np.int64(A1 > 0))
    dW1 = 1./m * np.dot(dZ1, X.T)
    db1 = 1./m * np.sum(dZ1, axis=1, keepdims = True)
    
    gradients = {"dZ3": dZ3, "dW3": dW3, "db3": db3,"dA2": dA2,
                 "dZ2": dZ2, "dW2": dW2, "db2": db2, "dA1": dA1, 
                 "dZ1": dZ1, "dW1": dW1, "db1": db1}
    
    return gradients

3 Gradient Checking 梯度检验

3.1 概念

• 计算