实现linear regression和logistic regression
import numpy as np
import sys
import math
def pro1():
sigma = 0.1
err = 0.008
d = 8.0
print (d + 1) / (1 - err / sigma / sigma)
def pro7():
eta = 0.01
def diff_u(u, v):
return math.exp(u) + math.exp(u * v) * v + 2 * u - 2 * v - 3
def diff_v(u, v):
return 2 * math.exp(2 * v) + u * math.exp(u * v) - 2 * u + 4 * v - 2
def Euv(u, v):
return math.exp(u) + math.exp(2 * v) + math.exp(u * v) + u * u - 2 * u * v + 2 * v * v - 3 * u - 2 * v
u = 0
v = 0
for i in range(0, 5):
delta_u = diff_u(u, v)
delta_v = diff_v(u, v)
u = u - eta * delta_u
v = v - eta * delta_v
print u, v, Euv(u, v)
def pro10():
def Euv(u, v):
return math.exp(u) + math.exp(2 * v) + math.exp(u * v) + u * u - 2 * u * v + 2 * v * v - 3 * u - 2 * v
def hessian(u, v):
a00 = math.exp(u) + 2 + v * v * math.exp(u * v)
a01 = u * v * math.exp(u * v) + math.exp(u * v) - 2
a10 = a01
a11 = 4 * math.exp(2 * v) + u * u * math.exp(u * v) + 4
return np.matrix([[a00, a01], [a10, a11]])
def gradient(u, v):
a = math.exp(u) + math.exp(u * v) * v + 2 * u - 2 * v - 3
b = 2 * math.exp(2 * v) + u * math.exp(u * v) - 2 * u + 4 * v - 2
return np.matrix([[a], [b]])
u = 0
v = 0
for i in range(0, 5):
hess = hessian(u, v)
print hess.shape
out = np.zeros(shape=(2, 1), dtype=float)
delta = np.matmul(np.linalg.inv(hess), gradient(u, v))
print u, v, delta, delta.shape, delta[0, 0], delta[1, 0] # if use delta[0][0], the shape is (1, 1)
u = u - delta[0, 0]
v = v - delta[1, 0]
# print u, v, delta, delta.shape, delta[0], delta[1]
print u, v, Euv(u, v)
def fun(x, y):
val = x * x + y * y - 0.6
if val >= 0:
return 1
else:
return -1
def pro13():
np.random.seed(1234)
N = 1000
Ein = []
for rep in range(0, 1000):
train_x = []
train_y = []
for i in range(0, N):
x1 = np.random.uniform(-1, 1)
x2 = np.random.uniform(-1, 1)
train_x.append([1, x1, x2])
label = fun(x1, x2)
if np.random.random() < 0.1:
label = -label
train_y.append([label])
x = np.matrix(train_x)
y = np.matrix(train_y)
# print np.linalg.inv(np.matmul(x.transpose(), x))
# print "====", np.matmul(np.linalg.inv(np.matmul(x.transpose(), x)), x.transpose()).shape
# print y.shape
# print x, x.transpose(), y
w_lin = np.matmul(np.matmul(np.linalg.inv(np.matmul(x.transpose(), x)), x.transpose()), y)
# test
# print w_lin.shape
predicted_y = np.matmul(x, w_lin)
for i in range(0, len(predicted_y)):
if predicted_y[i] >= 0:
predicted_y[i] = 1
else:
predicted_y[i] = -1
diff = [predicted_y[i] != y[i] for i in range(0, len(y))]
error = 1.0 * np.count_nonzero(diff) / len(train_x)
print rep, error
Ein.append(error)
print "ave Ein", np.average(Ein)
def pro14():
w_cand = [[-1, -0.05, 0.08, 0.13, 15, 1.5],
[-1, -1.5, 0.08, 0.13, 0.05, 1.5],
[-1, 0.05, 0.08, 0.13, 1.5, 1.5],
[-1, -0.05, 0.08, 0.13, 1.5, 15],
[-1, -1.5, 0.08, 0.13, 0.05, 0.05]]
agree_num = [[] for i in range(0, 5)]
np.random.seed(1234)
N = 1000
Eout = []
for rep in range(0, 1000):
train_x = []
train_y = []
test_x = []
test_y = []
for i in range(0, N): # generate training data
x1 = np.random.uniform(-1, 1)
x2 = np.random.uniform(-1, 1)
train_x.append([1, x1, x2, x1 * x2, x1 * x1, x2 * x2])
label = fun(x1, x2)
if np.random.random() < 0.1:
label = -label
train_y.append([label])
for i in range(0, N): # generate testing data
x1 = np.random.uniform(-1, 1)
x2 = np.random.uniform(-1, 1)
test_x.append([1, x1, x2, x1 * x2, x1 * x1, x2 * x2])
label = fun(x1, x2)
if np.random.random() < 0.1:
label = -label
test_y.append([label])
x = np.matrix(train_x)
y = np.matrix(train_y)
w_lin = np.matmul(np.matmul(np.linalg.inv(np.matmul(x.transpose(), x)), x.transpose()), y)
predicted_y = np.matmul(np.matrix(test_x), w_lin)
for i in range(0, len(predicted_y)):
if predicted_y[i] >= 0:
predicted_y[i] = 1
else:
predicted_y[i] = -1
diff = [predicted_y[i] != test_y[i] for i in range(0, len(test_y))]
error = 1.0 * np.count_nonzero(diff) / len(test_x)
print rep, error
Eout.append(error)
# predicted_y = np.matmul(x, w_lin)
# for i in range(0, len(predicted_y)):
# if predicted_y[i] >= 0:
# predicted_y[i] = 1
# else:
# predicted_y[i] = -1
# for cand in range(0, 5):
# predicted_cand = np.matmul(x, np.matrix(w_cand[cand]).transpose())
# for i in range(0, len(predicted_cand)):
# if predicted_cand[i] >= 0:
# predicted_cand[i] = 1
# else:
# predicted_cand[i] = -1
# agree = [predicted_y[i] == predicted_cand[i] for i in range(0, len(predicted_cand))]
# agree_num[cand].append(agree)
# print rep
# for i in range(0, 5):
# print "candidate", i, np.average(agree_num[i])
print "ave Eout", np.average(Eout)
def load_data(filename):
x = []
y = []
try:
fin = open(filename, 'r')
except IOError:
return
for data in fin:
data = data.strip("\n")
data = data.strip("\r")
data = data.strip(" ")
items = data.split(" ")
# print items
y.append(float(items[-1]))
tmp = []
for i in range(0, len(items) - 1):
tmp.append(float(items[i]))
x.append(tmp)
fin.close()
print x[0], y[0]
return np.array(x), np.array(y)
def pro18(SGD=False):
eta = 0.01
T = 2000
train_file = "hw3_train.dat"
test_file = "hw3_test.dat"
train_x, train_y = load_data(train_file)
test_x, test_y = load_data(test_file)
dim = len(train_x[0])
N = len(train_x)
np.random.seed(1234)
# w = np.random.random(size=dim)
w = np.zeros(shape=dim, dtype=float)
print train_x.shape, train_y.shape
n = 0
for t in range(0, T):
if SGD is True:
s = - train_y[n] * np.dot(w, train_x[n])
gradient = - 1.0 / (1 + np.exp(- s)) * train_y[n] * train_x[n]
else:
grad = []
# gradient
s = - train_y * np.dot(train_x, w) # dot return [w^T * train_x[i],...]
weighted_vec = 1.0 / (1 + np.exp(- s))
# print weighted_vec.shape
for i in range(0, N):
# s = - train_y[i] * np.dot(w, train_x[i])
# tmp = - 1.0 / (1 + np.exp(- s)) * train_y[i] * train_x[i]
# print tmp
tmp = - weighted_vec[i] * train_y[i] * train_x[i]
# if i == 0 and t == 0:
# print tmp, tmp.shape
grad.append(tmp)
gradient = np.average(grad, axis=0)
# print gradient.shape, w.shape
# print w, gradient
w = w - eta * gradient
# print w
print w
# test
s = np.dot(test_x, w) # dot return [w^T * train_x[i],...]
weighted_vec = 1.0 / (1 + np.exp(-s))
print weighted_vec.shape
predicted_y = np.where(weighted_vec >= 0.5, 1.0, -1.0)#[weighted_vec[i] >= 0.5 for i in range(0, len(weighted_vec))] # wrong benchmark make the Eout become 0.5
diff = [predicted_y[i] != test_y[i] for i in range(0, len(test_y))]
error = 1.0 * np.count_nonzero(diff) / len(test_x)
print "Eout", error
if __name__ == '__main__':
# pro18(True)
# pro13()
pro14()