徒手实现的神经网络-CSDN博客

个人手工实现的神经网络，可以训练出XOR逻辑模型。拷贝到本地后可以在CUP上训练。
随机初始化的weights对是否能训练成功有影响，可以跑多次进行测试。

# learning XOR throw a two layer network use gradient descent
# w 的初始化值对是否能训练成功和训练的速度都有较大影响。这是无限模型带来的问题
import math
import random
import numpy as np
from datetime import datetime

# random.seed(3)
random.seed(datetime.now())

weightSet = set()


def random_weight():
    while (True):
        w = (random.randint(-10, 10)) / 10
        if w == 0:
            continue
        if w not in weightSet:
            weightSet.add(w)
            return w


class Weights:
    indexMap = {"b1": 0, "w11": 1, "w12": 2, "b2": 3, "w21": 4, "w22": 5, "c": 6, "v1": 7, "v2": 8}
    weights = []

    def __init__(self):
        self.weights = [-1.5, 1, 1, 0.5, -1, -1, 0.5, -1, -1]

    def getIndex(self, name):
        return self.indexMap[name]

    def get(self, index):
        return self.weights[index]

    def getWeights(self):
        return self.weights

    def update(self, index, w):
        self.weights[index] = w


η = 0.05

weights = Weights()
for i in range(len(weights.getWeights())):
    weights.update(i, random_weight())

# "b1", "w11", "w12", "b2", "w21", "w22", "c", "v1", "v2"
# weights.weights = [-1.5, 1, 1, 0.5, -1, -1, 0.5, -1, -1]  # target
# target:
# weights.weights = [-3.6256294452213225, 2.472015878775767, 2.472015878775767, 1.4097549175948436, -3.0465380385550866,
#                    -3.0465380385550866, -3.7807463009027975, -4.221045218825591, -4.194984747863033]

# weights.weights = [0.5,-1,1,0.5,-1,-1,1.5,-1,-1]
# weights.weights = [-1.0, 0, 1, 0.5, -1, -1, 0.5, -1, -1]
# weights.weights = [0.0, 0.0, 0.0,  0.0, 0.0,  0.0, 0.0, 0.0, 0.0] # 不能初始化为0

inputs = [[1, 1], [1, 0], [0, 1], [0, 0]]
ts = [0, 1, 1, 0]


def cost():
    return


λ = 0.001  # weight decay λ


def pW(weights, λ):
    r = 0
    for w in weights.weights:
        r = r + w * w
    r = (r * λ) / 2
    return r


def sigmoid(num):
    # return step(num)
    # num = num * 5
    r = 1 / (1 + math.exp(-num))
    return r


def tanh(num) -> object:
    r = 2 / (1 + math.exp(-2 * num)) - 1
    return r


def step(num):
    if num > 0:
        return 1
    else:
        return 0


def activation(num):
    # if(num)>0:
    #     return 1
    # else:
    #     return 0
    return sigmoid(num)


def calculate_s(inputs, weights):
    result = weights[0]
    for i in range(len(inputs)):
        x = inputs[i]
        result = result + x * weights[i + 1]
    return result


def c_z(x1, x2, weights):
    b1, w11, w12, b2, w21, w22, c, v1, v2 = weights.getWeights()

    y1 = tanh(x1 * w11 + x2 * w12 + b1)
    y2 = tanh(x1 * w21 + x2 * w22 + b2)
    s = y1 * v1 + y2 * v2 + c
    z = activation(s)
    return z


def c_z_step(x1, x2, weights):
    b1, w11, w12, b2, w21, w22, c, v1, v2 = weights.getWeights()

    y1 = (x1 * w11 + x2 * w12 + b1)
    if y1 > 0:
        y1 = 1
    else:
        y1 = 0
    y2 = (x1 * w21 + x2 * w22 + b2)
    if y2 > 0:
        y2 = 1
    else:
        y2 = 0
    s = y1 * v1 + y2 * v2 + c
    if s > 0:
        s = 1
    else:
        s = 0
    return s


def calculate_dE_dz(t, z, weights):
    dE_dz = z - t
    # dE_dz = dE_dz + λ * sum(weights.weights)
    return dE_dz


def calculatePartialDerivatives_c(t, x1, x2, weights):
    b1, w11, w12, b2, w21, w22, c, v1, v2 = weights.getWeights()

    y1 = tanh(x1 * w11 + x2 * w12 + b1)
    y2 = tanh(x1 * w21 + x2 * w22 + b2)
    s = y1 * v1 + y2 * v2 + c
    z = activation(s)
    dE_dz = calculate_dE_dz(t, z, weights)
    dz_ds = z * (1 - z)
    dE_ds = dE_dz * dz_ds
    dE_dc = dE_ds
    return dE_dc


def calculatePartialDerivatives_v1(t, x1, x2, weights):
    b1, w11, w12, b2, w21, w22, c, v1, v2 = weights.getWeights()

    y1 = tanh(x1 * w11 + x2 * w12 + b1)
    y2 = tanh(x1 * w21 + x2 * w22 + b2)
    s = y1 * v1 + y2 * v2 + c
    z = activation(s)
    dE_dz = calculate_dE_dz(t, z, weights)
    dz_ds = z * (1 - z)
    dE_ds = dE_dz * dz_ds
    dE_dv1 = dE_ds * y1
    return dE_dv1


def calculatePartialDerivatives_v2(t, x1, x2, weights):
    b1, w11, w12, b2, w21, w22, c, v1, v2 = weights.getWeights()

    y1 = tanh(x1 * w11 + x2 * w12 + b1)
    y2 = tanh(x1 * w21 + x2 * w22 + b2)
    s = y1 * v1 + y2 * v2 + c
    z = activation(s)
    dE_dz = calculate_dE_dz(t, z, weights)
    dz_ds = z * (1 - z)
    dE_ds = dE_dz * dz_ds
    dout = dE_ds
    dE_dv2 = dout * y2
    return dE_dv2


def calculatePartialDerivatives_b1(t, x1, x2, weights):
    b1, w11, w12, b2, w21, w22, c, v1, v2 = weights.getWeights()

    y1 = tanh(x1 * w11 + x2 * w12 + b1)
    y2 = tanh(x1 * w21 + x2 * w22 + b2)
    s = y1 * v1 + y2 * v2 + c
    z = activation(s)
    dE_dz = calculate_dE_dz(t, z, weights)
    dz_ds = z * (1 - z)
    dE_ds = dE_dz * dz_ds
    dout = dE_ds
    dE_dy1 = dout * v1
    dy1_du1 = 1 - y1 ** 2
    dE_du1 = dE_dy1 * dy1_du1
    du1_db1 = 1
    dE_db1 = dE_du1 * du1_db1
    return dE_db1


def calculatePartialDerivatives_w11(t, x1, x2, weights):
    b1, w11, w12, b2, w21, w22, c, v1, v2 = weights.getWeights()

    y1 = tanh(x1 * w11 + x2 * w12 + b1)
    y2 = tanh(x1 * w21 + x2 * w22 + b2)
    s = y1 * v1 + y2 * v2 + c

    z = activation(s)
    dE_dz = calculate_dE_dz(t, z, weights)
    dz_ds = z * (1 - z)
    dE_ds = dE_dz * dz_ds
    dout = dE_ds

    dE_dy1 = dout * v1

    dy1_du1 = 1 - y1 ** 2
    dE_du1 = dE_dy1 * dy1_du1

    du1_dw11 = x1
    dE_dw11 = dE_du1 * du1_dw11

    return dE_dw11


def calculatePartialDerivatives_w12(t, x1, x2, weights):
    b1, w11, w12, b2, w21, w22, c, v1, v2 = weights.getWeights()

    y1 = tanh(x1 * w11 + x2 * w12 + b1)
    y2 = tanh(x1 * w21 + x2 * w22 + b2)
    s = y1 * v1 + y2 * v2 + c

    z = activation(s)
    dE_dz = calculate_dE_dz(t, z, weights)
    dz_ds = z * (1 - z)
    dE_ds = dE_dz * dz_ds
    dout = dE_ds

    dE_dy1 = dout * v1

    dy1_du1 = 1 - y1 ** 2
    dE_du1 = dE_dy1 * dy1_du1

    du1_dw12 = x2
    dE_dw12 = dE_du1 * du1_dw12

    return dE_dw12


def calculatePartialDerivatives_b2(t, x1, x2, weights):
    b1, w11, w12, b2, w21, w22, c, v1, v2 = weights.getWeights()

    y1 = tanh(x1 * w11 + x2 * w12 + b1)
    y2 = tanh(x1 * w21 + x2 * w22 + b2)
    s = y1 * v1 + y2 * v2 + c
    z = activation(s)
    dE_dz = calculate_dE_dz(t, z, weights)
    dz_ds = z * (1 - z)
    dE_ds = dE_dz * dz_ds
    dout = dE_ds
    dE_dy2 = dout * v2
    dy2_du2 = 1 - y2 ** 2
    dE_du2 = dE_dy2 * dy2_du2
    du2_db2 = 1
    dE_db2 = dE_du2 * du2_db2
    return dE_db2


def calculatePartialDerivatives_w21(t, x1, x2, weights):
    b1, w11, w12, b2, w21, w22, c, v1, v2 = weights.getWeights()

    y1 = tanh(x1 * w11 + x2 * w12 + b1)
    y2 = tanh(x1 * w21 + x2 * w22 + b2)
    s = y1 * v1 + y2 * v2 + c
    z = activation(s)
    dE_dz = calculate_dE_dz(t, z, weights)
    dz_ds = z * (1 - z)
    dE_ds = dE_dz * dz_ds
    dout = dE_ds
    dE_dy2 = dout * v2
    dy2_du2 = 1 - y2 ** 2
    dE_du2 = dE_dy2 * dy2_du2
    du2_dw21 = x1
    dE_dw21 = dE_du2 * du2_dw21

    return dE_dw21


def calculatePartialDerivatives_w22(t, x1, x2, weights):
    b1, w11, w12, b2, w21, w22, c, v1, v2 = weights.getWeights()

    y1 = tanh(x1 * w11 + x2 * w12 + b1)
    y2 = tanh(x1 * w21 + x2 * w22 + b2)
    s = y1 * v1 + y2 * v2 + c
    z = activation(s)
    dE_dz = calculate_dE_dz(t, z, weights)
    dz_ds = z * (1 - z)
    dE_ds = dE_dz * dz_ds
    dout = dE_ds
    dE_dy2 = dout * v2
    dy2_du2 = 1 - y2 ** 2
    dE_du2 = dE_dy2 * dy2_du2
    du2_dw22 = x2
    dE_dw22 = dE_du2 * du2_dw22
    return dE_dw22


def calculatePartialDerivatives_sum(inputs, weights, calculatePartialDerivatives_single):
    d = 0
    for i in range(len(inputs)):
        input = inputs[i]
        t = ts[i]
        x1 = input[0]
        x2 = input[1]
        d = d + calculatePartialDerivatives_single(t, x1, x2, weights)
        # print(x1, x2, d)
    return d


calulatePartialDerivativesFunctions = [
    calculatePartialDerivatives_b1,
    calculatePartialDerivatives_w11,
    calculatePartialDerivatives_w12,
    calculatePartialDerivatives_b2,
    calculatePartialDerivatives_w21,
    calculatePartialDerivatives_w22,
    calculatePartialDerivatives_c,
    calculatePartialDerivatives_v1,
    calculatePartialDerivatives_v2]


def getCalulatePartialDerivativesFunction(index):
    return calulatePartialDerivativesFunctions[index]


def calulateError(inputs, ts, weightIndex, w, weights):
    weights.update(weightIndex, w)
    # print(weights.__dict__)
    E = 0
    for i in range(len(inputs)):
        input = inputs[i]
        t = ts[i]
        x1 = input[0]
        x2 = input[1]
        q = c_z(x1, x2, weights)

        E = E + ((q - t) ** 2)

        # p = activation(t)
        # D_KL = (p * (math.log(p, 2) - math.log(q, 2)))
        # E = E + D_KL

        # print((z-t), (z-t)**2)
    E = E / 2
    # E = E + pW(weights, λ)  # Weight decay
    return E


epochs = 10000
# epochs = 10


def adjustWeights():
    weightNames = ["b1", "w11", "w12", "b2", "w21", "w22", "c", "v1", "v2"]
    # weightNames = ["w11", "w12"]
    weightIndexs = []
    for weightName in weightNames:
        i = weights.getIndex(weightName)
        weightIndexs.append(i)
    derivatives = [0] * 9
    for i in range(epochs):
        # for i in range(1):
        # 计算完所有的derivatives之后再更新ws或者每次都更新每个w都可以。
        # 不能保证解决平原，马鞍等问题。但因为初始的w都比较小，从而很大概率？上避开了这些问题
        for weightIndex in weightIndexs:
            cF = getCalulatePartialDerivativesFunction(weightIndex)
            d = calculatePartialDerivatives_sum(inputs, weights, cF)
            derivatives[weightIndex] = d
            # beforeW = weights.get(weightIndex)
            # weights.update(weightIndex, beforeW - η * d)

        for weightIndex in weightIndexs:
            d = derivatives[weightIndex]
            beforeW = weights.get(weightIndex)
            weights.update(weightIndex, beforeW - η * d)
        print("middle:", weights.__dict__)


def check():
    E = 0
    for i in range(len(inputs)):
        input = inputs[i]
        t = ts[i]
        x1 = input[0]
        x2 = input[1]
        z = c_z(x1, x2, weights)
        E = E + (z - t) ** 2
        print("[{0},{1}] target: {2}, actual:{3}, E:{4}".format(x1, x2, t, z, E))
    print("E", E / 2)


if __name__ == '__main__':
    # weights.update(weights.getIndex("w21"), 10)
    print("start:", weights.__dict__)
    # check()
    adjustWeights()
    print("final:", weights.__dict__)
    check()