李宏毅2020机器学习HW_2:Classification（via.MBGD)

"""
author: Andrew_Wang
date:(start): 2021/4/8 15:57
Algorithm：MBGD(Mini-batch)
"""

import numpy as np
import matplotlib.pyplot as plt

"""
_shuffle：打乱数据顺序，类似于重新洗牌，进行分批次训练（即每次将一部分数据给模型进行训练，计算损失）
_sigmoid：激活函数
_f（X，w，b）：向前传播，计算激活值
_predict(X, w, b)：预测
_accuracy(Y_pred, Y_label)：计算准确度
_cross_entropy_loss：交叉熵损失函数
_gradient：计算梯度值，用于更新w和b
"""
np.random.seed(0)


# 归一化
def normalize(X, train=True, specified_column=None, X_mean=None, X_std=None):
    # This function normalizes specific columns of X.
    # The mean and standard variance of training data will be reused when processing testing data.
    #
    # Arguments:
    #     X: data to be processed
    #     train: 'True' when processing training data, 'False' for testing data
    #     specific_column: indexes of the columns that will be normalized. If 'None', all columns
    #         will be normalized.
    #     X_mean: mean value of training data, used when train = 'False'
    #     X_std: standard deviation of training data, used when train = 'False'
    # Outputs:
    #     X: normalized data
    #     X_mean: computed mean value of training data
    #     X_std: computed standard deviation of training data

    if specified_column == None:
        # 为每个数据添加索值
        specified_column = np.arange(X.shape[1])
    if train:
        # 求取每个数据的平均值和标准差
        X_mean = np.mean(X[:, specified_column], 0).reshape(1, -1)
        X_std = np.std(X[:, specified_column], 0).reshape(1, -1)
    # 归一化数据
    X[:, specified_column] = (X[:, specified_column] - X_mean) / (X_std + 1e-8)
    # 返回归一化后的数据，均值，标准差
    return X, X_mean, X_std


# 分割训练集
def train_dev_split(X, Y, dev_ratio=0.25):
    # This function spilts data into training set and development set.
    train_size = int(len(X) * (1 - dev_ratio))
    return X[:train_size], Y[:train_size], X[train_size:], Y[train_size:]


# 打乱数据顺序，重新为minibatch分配
def shuffle(X, Y):
    # This function shuffles two equal-length list/array, X and Y, together.
    randomize = np.arange(len(X))
    np.random.shuffle(randomize)
    return X[randomize], Y[randomize]


# sigmoid函数
def sigmoid(z):
    #     # Sigmoid function can be used to calculate probability.
    #     # To avoid overflow, minimum/maximum output value is set.
    #     # np.clip()函数用来截取最大最小值，np.clip(s, 1e-8, (1 - 1e-8))
    #     # 表示比1e-8小的值都赋值为1e-8，比(1 - 1e-8)大的值都赋值为1 - 1e-8
    res = np.exp(z) / (1 + np.exp(z))
    return np.clip(res, 1e-8, (1 - (1e-8)))


def forward(X, w, b):
    # This is the logistic regression function, parameterized by w and b
    #
    # Arguements:
    #     X: input data, shape = [batch_size, data_dimension]
    #     w: weight vector, shape = [data_dimension, ]
    #     b: bias, scalar
    # Output:
    #     predicted probability of each row of X being positively labeled, shape = [batch_size, ]
    return sigmoid(np.matmul(X, w) + b)


# 对结果进行预测
def predict(X, w, b):
    return np.round(forward(X, w, b)).astype(np.int)


def accurency(Y_pred, Y_label):
    # This function calculates prediction accuracy
    acc = 1 - np.mean(np.abs(Y_pred - Y_label))
    return acc


# 交叉熵损失函数
def cross_entropy_loss(y_pred, Y_label):
    # This function computes the cross entropy.
    #
    # Arguments:
    #     y_pred: probabilistic predictions, float vector
    #     Y_label: ground truth labels, bool vector
    # Output:
    #     cross entropy, scalar
    cross_entropy = -np.dot(Y_label, np.log(y_pred)) - np.dot((1 - Y_label), np.log(1 - y_pred))
    return cross_entropy


# 计算梯度值(?)
def gradient(X, Y_label, w, b):
    # This function computes the gradient of cross entropy loss with respect to weight w and bias b.
    y_pred = forward(X, w, b)
    pred_error = Y_label - y_pred
    w_grad = -np.sum(pred_error * X.T, 1)
    b_grad = -np.sum(pred_error)
    return w_grad, b_grad


if __name__ == '__main__':
    # 加载数据
    with open('X_train') as f:
        next(f)
        X_train = np.array([line.strip('\n').split(',')[1:] for line in f], dtype=float)
    with open('Y_train') as f:
        next(f)
        Y_train = np.array([line.strip('\n').split(',')[1] for line in f], dtype=float)
    with open('X_test') as f:
        next(f)
        X_test = np.array([line.strip('\n').split(',')[1:] for line in f], dtype=float)

    # 归一化
    X_train, X_mean, X_std = normalize(X_train, train=True)
    # X_test = normalize(X_test, train=False, specified_column=None, X_mean=X_mean, X_std=X_std)
    X_test, _, _ = normalize(X_test, train=False, specified_column=None, X_mean=X_mean, X_std=X_std)

    # 划分训练集和验证集
    dev_ratio = 0.1
    X_train, Y_train, X_dev, Y_dev = train_dev_split(X_train, Y_train, dev_ratio=dev_ratio)
    train_size = X_train.shape[0]
    dev_size = X_dev.shape[0]
    test_size = X_test.shape[0]
    data_dim = X_train.shape[1]
    print('Size of training set: {}'.format(train_size))
    print('Size of development set: {}'.format(dev_size))
    print('Size of testing set: {}'.format(test_size))
    print('Dimension of data: {}'.format(data_dim))

    # # 开始训练
    # w = np.zeros(data_dim)
    # b = np.zeros(1)
    # # 设置其他超参数（迭代次数，分批次大小，学习率）
    # iter_time = 100
    # batch_size = 8
    # learning_rate = 0.2
    # # 创建列表用来保存训练集和验证集的损失值和准确度
    # train_loss = []
    # dev_loss = []
    # train_acc = []
    # dev_acc = []
    #
    # # 用来更新学习率
    # step = 1
    #
    # for epoch in range(iter_time):
    #     # 每个epoch都会重新洗牌
    #     X_train, Y_train = shuffle(X_train, Y_train)
    #     # 分批次进行训练w 和 b
    #     for idx in range(int(np.floor(train_size / batch_size))):
    #         X = X_train[idx * batch_size: (idx + 1) * batch_size]
    #         Y = Y_train[idx * batch_size: (idx + 1) * batch_size]
    #
    #         # 计算梯度值
    #         w_grad, b_grad = gradient(X, Y, w, b)
    #
    #         # 更新参数w 和 b
    #         w = w - learning_rate * w_grad
    #         b = b - learning_rate * b_grad
    #
    #         step = step + 1
    #
    #     # 参数总共更新了 iter_time × （train_size/batch_size）次
    #     # 计算训练集的损失值和准确度
    #     y_train_pred = forward(X_train, w, b)
    #     Y_train_pred = np.round(y_train_pred)
    #     train_acc.append(accurency(Y_train_pred, Y_train))
    #     train_loss.append(cross_entropy_loss(Y_train_pred, Y_train) / train_size)
    #     # 计算验证集的损失值和准确度
    #     y_dev_pred = forward(X_dev, w, b)
    #     Y_dev_pred = np.round(y_dev_pred)
    #     dev_acc.append(accurency(Y_dev_pred, Y_dev))
    #     dev_loss.append(cross_entropy_loss(Y_dev_pred, Y_dev) / dev_size)
    #
    #
    # print('Training loss: {}'.format(train_loss[-1]))
    # print('Development loss: {}'.format(dev_loss[-1]))
    # print('Training accuracy: {}'.format(train_acc[-1]))
    # print('Development accuracy: {}'.format(dev_acc[-1]))
    # 将w和b初始化为0
    w = np.zeros((data_dim,))
    b = np.zeros((1,))

    # 设置其他超参数（迭代次数，分批次大小，学习率）
    max_iter = 100
    batch_size = 5
    learning_rate = 0.3

    # 创建列表用来保存训练集和验证集的损失值和准确度
    train_loss = []
    dev_loss = []
    train_acc = []
    dev_acc = []

    # 用来更新学习率
    step = 1

    # 训练
    for epoch in range(max_iter):
        # 每个epoch都会重新洗牌
        X_train, Y_train = shuffle(X_train, Y_train)

        # 分批次训练
        for idx in range(int(np.floor(train_size / batch_size))):
            X = X_train[idx * batch_size:(idx + 1) * batch_size]
            Y = Y_train[idx * batch_size:(idx + 1) * batch_size]

            # 计算梯度值
            w_grad, b_grad = gradient(X, Y, w, b)

            # 更新参数w和b
            # 学习率随着迭代时间增加而减少
            w = w - learning_rate / np.sqrt(step) * w_grad
            b = b - learning_rate / np.sqrt(step) * b_grad

            step = step + 1
        # 参数总共更新了max_iter × （train_size/batch_size）次
        # 计算训练集的损失值和准确度
        y_train_pred = forward(X_train, w, b)
        Y_train_pred = np.round(y_train_pred)
        train_acc.append(accurency(Y_train_pred, Y_train))
        train_loss.append(cross_entropy_loss(y_train_pred, Y_train) / train_size)
        # 计算验证集的损失值和准确度
        y_dev_pred = forward(X_dev, w, b)
        Y_dev_pred = np.round(y_dev_pred)
        dev_acc.append(accurency(Y_dev_pred, Y_dev))
        dev_loss.append(cross_entropy_loss(y_dev_pred, Y_dev) / dev_size)

    print('Training loss: {}'.format(train_loss[-1]))
    print('Development loss: {}'.format(dev_loss[-1]))
    print('Training accuracy: {}'.format(train_acc[-1]))
    print('Development accuracy: {}'.format(dev_acc[-1]))

    # 绘图
    # Loss curve
    plt.plot(train_loss)
    plt.plot(dev_loss)
    plt.title('Loss')
    plt.legend(['train', 'dev'])
    plt.savefig('loss.png')
    plt.show()

    # Accuracy curve
    plt.plot(train_acc)
    plt.plot(dev_acc)
    plt.title('Accuracy')
    plt.legend(['train', 'dev'])
    plt.savefig('acc.png')
    plt.show()

    # 在测试集上进行预测
    output_fpath = '/Users/shuaishuairan/PycharmProjects/pythonProject3/ML_HW/output_logistic.csv'
    predictions = predict(X_test, w, b)
    with open('/Users/shuaishuairan/PycharmProjects/pythonProject3/ML_HW/output_logistic.csv', 'w') as f:
        f.write('id,label\n')
        for i, label in enumerate(predictions):
            f.write('{},{}\n'.format(i, label))

    # Print out the most significant weights