66、线性模型在梯度下降、tf回归、MLP优化求解表现

# -*- coding: utf-8 -*-
import tensorflow as tf
import random as rd
import math
import numpy as np
tf.compat.v1.disable_eager_execution()
se = tf.compat.v1.Session()
rd.seed(10)

def sample_generate(n, low, sup, w, e):
    X = [[(sup - low) * rd.random() + low if j < len(w) - 1 else 1 for j in range(len(w))] for i in range(n)]
    Y = [sum([w[j] * X[i][j] for j in range(len(w))]) + e * (rd.random() - 0.5) for i in range(n)]
    return X, Y

def f(w, X, Y):
    return (w.T * X.T * X * w - 2 * Y.T * X * w)[0, 0]

def mod(dw):
    return math.sqrt(sum([dw[i, 0] * dw[i, 0] for i in range(dw.size)]))

def Gradient(X, Y, r=0.8, step_min=0.00001):
    d = len(X[0])
    X = np.mat(X)
    Y = np.mat([[Y[i]] for i in range(len(Y))])
    w = [[0] for i in range(d)]
    w = np.mat(w)

    # 先采用放缩法确定寻优区间，由二分法确定寻优步长
    while True:
        left = 0
        right = 1
        dw = -X.T * X * w + X.T * Y
        while f(w + right * dw, X, Y) < f(w, X, Y):
            right = right * (2 - r)

        mid1 = left * 2 / 3 + right / 3
        mid2 = left / 3 + right * 2 / 3
        while abs(left - right) * mod(dw) > step_min:
            if f(w + mid1 * dw, X, Y) < f(w + mid2 * dw, X, Y):
                right = mid2
            else:
                left = mid1
            mid1 = left * 2 / 3 + right / 3
            mid2 = left / 3 + right * 2 / 3

        if left * mod(dw) < step_min:
            break
        w = w + left * dw

    w = w.tolist()
    return [w[i][0] for i in range(len(w))]

def train(X_train, Y_train):
    X = tf.compat.v1.placeholder(tf.float32, name='X')
    Y = tf.compat.v1.placeholder(tf.float32, name='Y')

    b = tf.Variable(0.0)
    w = tf.Variable(0.0)
    Y_hat = X*w+b

    loss = tf.square(Y-Y_hat, name='loss')
    #梯度下降
    # optimizer = tf.compat.v1.train.GradientDescentOptimizer(learning_rate=0.01).minimize(loss)
    #ADAM
    optimizer = tf.compat.v1.train.AdamOptimizer(learning_rate=0.01).minimize(loss)

    init_op = tf.compat.v1.global_variables_initializer()
    total = []

    se.run(init_op)
    # writer = tf.compat.v1.summary.FileWriter('graphs', se.graph)

    for i in range(10):
        total_loss = 0
        for x,y in zip(X_train, Y_train):
            _, l = se.run([optimizer, loss], feed_dict={X:x[0], Y:y})
            total_loss += l
        total.append(total_loss/len(X_train))
        print('Epoch{}: Loss {}'.format(i, total_loss/len(X_train)))
    # writer.close()
    w_value, b_value = se.run([w, b])
    return w_value, b_value, total[-1]

def MLP(X, Y, nums):

    X = [[x[0]] for x in X]
    Y = [[y] for y in Y]

    #神经网络参数
    L_w = []
    L_b = []
    nums = [len(X[0])] + nums + [1]
    for i in range(len(nums)-1):
        L_w.append(tf.Variable(tf.compat.v1.random_normal([nums[i], nums[i+1]], stddev=1, seed=1)))
        L_b.append(tf.Variable(tf.compat.v1.random_normal([nums[i+1]], stddev=1, seed=1)))
        # L_w.append(tf.Variable(tf.compat.v1.zeros([nums[i], nums[i + 1]])))
        # L_b.append(tf.Variable(tf.compat.v1.zeros([nums[i + 1]])))

    #占位符
    x = tf.compat.v1.placeholder(tf.float32, shape=(None, len(X[0])), name="x-input")
    y_ = tf.compat.v1.placeholder(tf.float32, shape=(None, 1), name="y-input")

    #前向传播
    L_a = []
    L_a.append(tf.matmul(x, L_w[0])+L_b[0])
    for i in range(1, len(L_w)):
        L_a.append(tf.matmul(L_a[i-1], L_w[i])+L_b[i])

    #定义损失函数，这里用交叉熵
    loss = tf.square(y_ - L_a[-1][0], name='loss')

    #优化器及学习率
    optimizer = tf.compat.v1.train.AdamOptimizer(learning_rate=0.01).minimize(loss)

    #变量初始化
    init_op = tf.compat.v1.global_variables_initializer()
    total = []

    se.run(init_op)

    for i in range(10):
        total_loss = 0
        for x1,y1 in zip(X, Y):
            _, l = se.run([optimizer, loss], feed_dict={x:[x1], y_:[y1]})
            total_loss += l[0][0]
        total.append(total_loss/len(X))
        print('Epoch{}: Loss {}'.format(i, total_loss/len(X)))

    return total[-1]


if __name__ == '__main__':

    X, Y = sample_generate(n=1000, low=-10, sup=10, w=[1, 2], e=0.2)

    #1-梯度下降法
    w1 = Gradient(X, Y)
    bias1 = sum([pow(sum([X[i][j]*w1[j] for j in range(len(w1))])-Y[i], 2) for i in range(len(X))])/len(X)

    #2-图化线性回归
    w, b, bias2 = train(X, Y)

    #3-MLP
    bias3 = MLP(X, Y, [3,3])