[深度学习] (1): 实现一个简单的XOR

摘要

开始学习DL，各种代价函数，各种分布，这里分享一个用NN实现XOR的例子

关键在于XOR必须使用一个非线性Activation (Relu / Tanh) 才能进行正确分类

---

输入输出

X = np.array([[0, 0], [0, 1], [1, 0], [1, 1]])
y = np.array([[0], [1], [1], [0]])

---

代价函数 J(Θ)

1. MSE
   作为最传统的代价函数， MSE简单效果也稳定
   

2. Cross Entropy
   引用DL书中的一句话来理解交叉熵：
   
   可以直观地理解到->CE与误差紧密相关.
   而且在使用梯度下降时，偏导数也非常好求，参考以下文章求CE的偏导数:
   http://www.cnblogs.com/python27/p/MachineLearningWeek05.html

---

示例代码

参考以下博客：http://www.cnblogs.com/Belter/p/6711160.html

代码中使用到的CE的推导下面都有
http://www.cnblogs.com/python27/p/MachineLearningWeek05.html

import numpy as np
import matplotlib.pyplot as plt
from scipy.special import expit

# Neural Network for XOR

HIDDEN_LAYER_SIZE = 2
INPUT_LAYER = 2  # input feature
NUM_LABELS = 1  # output class number
X = np.array([[0, 0], [0, 1], [1, 0], [1, 1]])
y = np.array([[0], [1], [1], [0]])

# 初始化权重，使得权重在(-epsilon, epsilon)之间
def rand_initialize_weights(L_in, L_out, epsilon):
    """
    Randomly initialize the weights of a layer with L_in
    incoming connections and L_out outgoing connections;

    Note that W should be set to a matrix of size(L_out, 1 + L_in) as
    the first column of W handles the "bias" terms
    """
    epsilon_init = epsilon
    W = np.random.rand(L_out, 1 + L_in) * 2 * epsilon_init - epsilon_init
    return W

def activate(x):
    return sigmoid(x)

def sigmoid(x):
    return expit(x)

def sigmoid_gradient(z):
    return np.multiply(sigmoid(z), (1 - sigmoid(z)))


# 使用Cross-Entropy作为二分类问题的代价函数
def nn_cost_function(theta1, theta2, X, y):
    m = X.shape[0] # m=4

    # 计算所有参数的偏导数/梯度
    D_1 = np.zeros(theta1.shape) # Δ_1
    D_2 = np.zeros(theta2.shape)  # Δ_2
    h_total = np.zeros((m, 1))  # 所有样本的预测值, m*1

    for t in range(m):
        a_1 = np.vstack((np.array([[1]]), X[t:t+1, :].T)) # 列向量 3*1 原始输入加上一个bias(1)
        z_2 = np.dot(theta1, a_1) # 2*1 # 用原始输入计算第二层的初始结果
        a_2 = np.vstack((np.array([[1]]), sigmoid(z_2))) # 列向量 3*1 原始输入加上一个bias(1)
        z_3 = np.dot(theta2, a_2) # 1*1
        a_3 = sigmoid(z_3)
        h = a_3 # hypothesis预测值h等于a_3
        h_total[t, 0] = h

        # 根据链式法则推导出每一层的误差
        delta_3 = h - y[t:t+1, :].T # 最后一层每一个单元的误差, δ_3, 1*1
        delta_2 = np.multiply(np.dot(theta2[:, 1:].T, delta_3), sigmoid_gradient(z_2))  # 第二层每一个单元的误差（不包括偏置单元）, δ_2, 2*1

        # 根据误差计算导数
        D_2 = D_2 + np.dot(delta_3, a_2.T)  # 第二层所有参数的误差, 1*3
        D_1 = D_1 + np.dot(delta_2, a_1.T)  # 第一层所有参数的误差, 2*3

    theta1_grad = (1.0 / m) * D_1  # 第一层参数的偏导数，取所有样本中参数的均值，没有加正则项
    theta2_grad = (1.0 / m) * D_2

    # 计算当前代价(交叉熵)的值
    J = (1.0 / m) * np.sum(-y * np.log(h_total) - (np.array([[1]]) - y) * np.log(1 - h_total))
    return {
        'theta1_grad': theta1_grad,
        'theta2_grad': theta2_grad,
        'J': J,
        'h': h_total
    }

theta1 = rand_initialize_weights(INPUT_LAYER, HIDDEN_LAYER_SIZE, epsilon=1)  # 2*3
theta2 = rand_initialize_weights(HIDDEN_LAYER_SIZE, NUM_LABELS, epsilon=1)   # 1*3

iter_times = 10000  # 之前的问题之二，迭代次数太少
alpha = 0.5  # 之前的问题之三，学习率太小
result = {'J': [], 'h': []}
theta_s = {}

for i in range(iter_times):
    cost_fun_result = nn_cost_function(theta1=theta1, theta2=theta2, X=X, y=y)
    theta1_g = cost_fun_result.get('theta1_grad')
    theta2_g = cost_fun_result.get('theta2_grad')
    J = cost_fun_result.get('J')
    h_current = cost_fun_result.get('h')

    # 梯度下降更新权重
    theta1 -= alpha * theta1_g
    theta2 -= alpha * theta2_g
    result['J'].append(J)
    result['h'].append(h_current)

plt.plot(result.get('J'))
plt.show()
print(result.get('h')[0], result.get('h')[-1])

---

使用Keras实现

# Implementation of XOR using NN in Keras

import numpy as np
import matplotlib.pyplot as plt

from keras.models import Sequential
from keras.layers import Dense, Activation

def main():

    # Preparing Data
    X = np.array([[0, 0], [1, 0], [0, 1], [1, 1]])
    y = np.array([0, 1, 1, 0])

    # Building Model
    model = Sequential()
    model.add(Dense(32, input_shape=(2, )))
    model.add(Activation('relu'))
    model.add(Dense(1))
    model.add(Activation('sigmoid'))
    model.compile(
        loss='binary_crossentropy',
        optimizer='sgd',
        metrics=['accuracy']
    )
    hist = model.fit(
        x=X,
        y=y,
        epochs=10000
    )
    plt.scatter(range(len(hist.history['loss'])), hist.history['loss'])
    plt.show()
    pass


if __name__ == '__main__':
    main()
    pass