Question 13-20 Homework 3 Machine Learning Foundation 机器学习基石 作业三 python code

实现linear regression和logistic regression

import numpy as np
import sys
import math


def pro1():
    sigma = 0.1
    err = 0.008
    d = 8.0
    print (d + 1) / (1 - err / sigma / sigma)

def pro7():
    eta = 0.01
    def diff_u(u, v):
        return math.exp(u) + math.exp(u * v) * v + 2 * u - 2 * v - 3
    def diff_v(u, v):
        return 2 * math.exp(2 * v) + u * math.exp(u * v) - 2 * u + 4 * v - 2
    def Euv(u, v):
        return math.exp(u) + math.exp(2 * v) + math.exp(u * v) + u * u - 2 * u * v + 2 * v * v - 3 * u - 2 * v
    u = 0
    v = 0
    for i in range(0, 5):
        delta_u = diff_u(u, v)
        delta_v = diff_v(u, v)
        u = u - eta * delta_u
        v = v - eta * delta_v
    print u, v, Euv(u, v)

def pro10():
    def Euv(u, v):
        return math.exp(u) + math.exp(2 * v) + math.exp(u * v) + u * u - 2 * u * v + 2 * v * v - 3 * u - 2 * v
    def hessian(u, v):
        a00 = math.exp(u) + 2 + v * v * math.exp(u * v)
        a01 = u * v * math.exp(u * v) + math.exp(u * v) - 2
        a10 = a01
        a11 = 4 * math.exp(2 * v) + u * u * math.exp(u * v) + 4
        return np.matrix([[a00, a01], [a10, a11]])
    def gradient(u, v):
        a = math.exp(u) + math.exp(u * v) * v + 2 * u - 2 * v - 3
        b = 2 * math.exp(2 * v) + u * math.exp(u * v) - 2 * u + 4 * v - 2
        return np.matrix([[a], [b]])
    u = 0
    v = 0
    for i in range(0, 5):
        hess = hessian(u, v)
        print hess.shape
        out = np.zeros(shape=(2, 1), dtype=float)
        delta = np.matmul(np.linalg.inv(hess), gradient(u, v))
        print u, v, delta, delta.shape, delta[0, 0], delta[1, 0] # if use delta[0][0], the shape is (1, 1)
        u = u - delta[0, 0]
        v = v - delta[1, 0]
        # print u, v, delta, delta.shape, delta[0], delta[1]
    print u, v, Euv(u, v)

def fun(x, y):
    val = x * x + y * y - 0.6
    if val >= 0:
        return 1
    else:
        return -1


def pro13():
    np.random.seed(1234)
    N = 1000
    Ein = []
    for rep in range(0, 1000):
        train_x = []
        train_y = []
        for i in range(0, N):
            x1 = np.random.uniform(-1, 1)
            x2 = np.random.uniform(-1, 1)
            train_x.append([1, x1, x2])
            label = fun(x1, x2)
            if np.random.random() < 0.1:
                label = -label
            train_y.append([label])
        x = np.matrix(train_x)
        y = np.matrix(train_y)
        # print np.linalg.inv(np.matmul(x.transpose(), x))
        # print "====", np.matmul(np.linalg.inv(np.matmul(x.transpose(), x)), x.transpose()).shape
        # print y.shape
        # print x, x.transpose(), y

        w_lin = np.matmul(np.matmul(np.linalg.inv(np.matmul(x.transpose(), x)), x.transpose()), y)
        # test
        # print w_lin.shape
        predicted_y = np.matmul(x, w_lin)
        for i in range(0, len(predicted_y)):
            if predicted_y[i] >= 0:
                predicted_y[i] = 1
            else:
                predicted_y[i] = -1
        diff = [predicted_y[i] != y[i] for i in range(0, len(y))]
        error = 1.0 * np.count_nonzero(diff) / len(train_x)
        print rep, error
        Ein.append(error)
    print "ave Ein", np.average(Ein)

def pro14():
    w_cand = [[-1, -0.05, 0.08, 0.13, 15, 1.5],
              [-1, -1.5, 0.08, 0.13, 0.05, 1.5],
              [-1, 0.05, 0.08, 0.13, 1.5, 1.5],
              [-1, -0.05, 0.08, 0.13, 1.5, 15],
              [-1, -1.5, 0.08, 0.13, 0.05, 0.05]]
    agree_num = [[] for i in range(0, 5)]
    np.random.seed(1234)
    N = 1000
    Eout = []
    for rep in range(0, 1000):
        train_x = []
        train_y = []
        test_x = []
        test_y = []
        for i in range(0, N):  # generate training data
            x1 = np.random.uniform(-1, 1)
            x2 = np.random.uniform(-1, 1)
            train_x.append([1, x1, x2, x1 * x2, x1 * x1, x2 * x2])
            label = fun(x1, x2)
            if np.random.random() < 0.1:
                label = -label
            train_y.append([label])
        for i in range(0, N):  # generate testing data
            x1 = np.random.uniform(-1, 1)
            x2 = np.random.uniform(-1, 1)
            test_x.append([1, x1, x2, x1 * x2, x1 * x1, x2 * x2])
            label = fun(x1, x2)
            if np.random.random() < 0.1:
                label = -label
            test_y.append([label])
        x = np.matrix(train_x)
        y = np.matrix(train_y)
        w_lin = np.matmul(np.matmul(np.linalg.inv(np.matmul(x.transpose(), x)), x.transpose()), y)
        predicted_y = np.matmul(np.matrix(test_x), w_lin)
        for i in range(0, len(predicted_y)):
            if predicted_y[i] >= 0:
                predicted_y[i] = 1
            else:
                predicted_y[i] = -1
        diff = [predicted_y[i] != test_y[i] for i in range(0, len(test_y))]
        error = 1.0 * np.count_nonzero(diff) / len(test_x)
        print rep, error
        Eout.append(error)
        # predicted_y = np.matmul(x, w_lin)
        # for i in range(0, len(predicted_y)):
        #     if predicted_y[i] >= 0:
        #         predicted_y[i] = 1
        #     else:
        #         predicted_y[i] = -1
        # for cand in range(0, 5):
        #     predicted_cand = np.matmul(x, np.matrix(w_cand[cand]).transpose())
        #     for i in range(0, len(predicted_cand)):
        #         if predicted_cand[i] >= 0:
        #             predicted_cand[i] = 1
        #         else:
        #             predicted_cand[i] = -1
        #     agree = [predicted_y[i] == predicted_cand[i] for i in range(0, len(predicted_cand))]
        #     agree_num[cand].append(agree)
        # print rep
    # for i in range(0, 5):
    #     print "candidate", i, np.average(agree_num[i])
    print "ave Eout", np.average(Eout)

def load_data(filename):
    x = []
    y = []
    try:
        fin = open(filename, 'r')
    except IOError:
        return
    for data in fin:
        data = data.strip("\n")
        data = data.strip("\r")
        data = data.strip(" ")
        items = data.split(" ")
        # print items
        y.append(float(items[-1]))
        tmp = []
        for i in range(0, len(items) - 1):
            tmp.append(float(items[i]))
        x.append(tmp)
    fin.close()
    print x[0], y[0]
    return np.array(x), np.array(y)


def pro18(SGD=False):
    eta = 0.01
    T = 2000
    train_file = "hw3_train.dat"
    test_file = "hw3_test.dat"
    train_x, train_y = load_data(train_file)
    test_x, test_y = load_data(test_file)
    dim = len(train_x[0])
    N = len(train_x)
    np.random.seed(1234)
    # w = np.random.random(size=dim)
    w = np.zeros(shape=dim, dtype=float)
    print train_x.shape, train_y.shape
    n = 0
    for t in range(0, T):
        if SGD is True:
            s = - train_y[n] * np.dot(w, train_x[n])
            gradient = - 1.0 / (1 + np.exp(- s)) * train_y[n] * train_x[n]

        else:
            grad = []
            # gradient
            s = - train_y * np.dot(train_x, w)  # dot return [w^T * train_x[i],...]
            weighted_vec = 1.0 / (1 + np.exp(- s))
            # print weighted_vec.shape
            for i in range(0, N):
                # s = - train_y[i] * np.dot(w, train_x[i])
                # tmp = - 1.0 / (1 + np.exp(- s)) * train_y[i] * train_x[i]
                # print tmp
                tmp = - weighted_vec[i] * train_y[i] * train_x[i]
                # if i == 0 and t == 0:
                #     print tmp, tmp.shape
                grad.append(tmp)
            gradient = np.average(grad, axis=0)
        # print gradient.shape, w.shape
        # print w, gradient
        w = w - eta * gradient
        # print w
    print w
    # test
    s = np.dot(test_x, w)  # dot return [w^T * train_x[i],...]
    weighted_vec = 1.0 / (1 + np.exp(-s))
    print weighted_vec.shape
    predicted_y = np.where(weighted_vec >= 0.5, 1.0, -1.0)#[weighted_vec[i] >= 0.5 for i in range(0, len(weighted_vec))] # wrong benchmark make the Eout become 0.5
    diff = [predicted_y[i] != test_y[i] for i in range(0, len(test_y))]
    error = 1.0 * np.count_nonzero(diff) / len(test_x)
    print "Eout", error
if __name__ == '__main__':
    # pro18(True)
    # pro13()
    pro14()