3.4 选择两个UCI数据集,比较10折交叉验证法和留一法所估计出的对率回归的错误率。
使用的UCI数据集是关于乳腺癌的。其描述为:
数据集链接为Breast Cancer Coimbra
代码
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
def getDataSet(filename):
data = pd.read_csv(filename)
dataSet = data.values # pd.DataFrame to np.array
dataArr = dataSet[:, :-1] # feature array
labelArr = dataSet[:, -1] # label array
# 标准化
# dataMean = dataArr.mean(axis=0)
dataMax = dataArr.max(axis=0)
dataMin = dataArr.min(axis=0)
dataArr = (dataArr - dataMax) / (dataMax - dataMin)
# 增加偏置参数
dataArr = np.insert(dataArr, 0,
np.ones(dataArr.shape[0]),
axis=1)
# 因为原来数据中1代表健康, 2代表病人,用逻辑回归要将2变为0
labelArr[labelArr == 2] = 0
return dataArr, labelArr
# # test getDataSet()
# dataArr, labelArr = getDataSet('dataR2.csv')
# print("dataArr's shape is ", dataArr.shape)
# print("labelArr's shape is ", labelArr.shape)
# print(labelArr)
def sigmoid(Z):
return 1.0 / (1 + np.exp(-Z))
def newton(dataArr, labelArr):
"""
calculate logistic parameters by newton method
:param dataArr: input data set with shape (m, n)
:param labelArr: the label of data set with shape (m, 1)
:return: returns the parameters obtained by newton method
"""
m, n = dataArr.shape
labelArr = labelArr.reshape(-1, 1)
beta = np.ones((n, 1))
errList = [] # save error history
z = np.dot(dataArr, beta)
oldLbeta = 0
# shape (m, 1)
newLBetaMat = -labelArr * z + np.log(1 + sigmoid(z))
newLBeta = np.sum(newLBetaMat)
it = 0
while abs(oldLbeta - newLBeta) > 1e-5:
it += 1
# py0 = p(y=0|x) with shape (m,1)
py0 = sigmoid(-np.dot(dataArr, beta))
py1 = 1 - py0
# 'reshape(n)' get shape (n,); 'np.diag' get diagonal matrix with shape (m,m)
p = np.diag((py0 * py1).reshape(m))
# shape (m,n)
dBetaMat = -dataArr * (labelArr - py1)
# first derivative with shape (1, n)
dBeta = np.sum(dBetaMat, axis=0, keepdims=True)
# second derivative with shape (n, n)
dBeta2 = dataArr.T.dot(p).dot(dataArr)
try:
dBeta2Inv = np.linalg.inv(dBeta2)
# (n,1) (n,1) (n,n) (n,1)
beta = beta - np.dot(dBeta2Inv, dBeta.T)
z = np.dot(dataArr, beta)
oldLbeta = newLBeta
newLBetaMat = -labelArr * z + np.log(1 + sigmoid(z))
newLBeta = np.sum(newLBetaMat)
pre = predict(beta, dataArr)
errorRate = cntErrRate(pre, labelArr)
errList.append(errorRate)
except:
print("maxtrix is singular, "
"now newton iteration is ", it)
break;
return beta, errList
def gradDescent(dataArr, labelArr, alpha, T):
"""
calculate logistic parameters by gradient descent
:param dataArr: input data set with shape (m, n)
:param labelArr: the label of data set with shape (m, 1)
:param alpha: step length (learning rate)
:param T: iteration
:return: parameters
"""
m, n = dataArr.shape
labelArr = labelArr.reshape(-1, 1)
errList = []
beta = np.ones((n, 1))
for t in range(T):
# py0 = p(y=1|x) with shape (m,1)
py1 = sigmoid(np.dot(dataArr, beta))
dBetaMat = -dataArr * (labelArr - py1)
# shape (1,n)
dBeta = np.sum(dBetaMat, axis=0, keepdims=True)
beta -= alpha * dBeta.T
# test code
# pre = predict(beta, dataArr)
# errorRate = cntErrRate(pre, labelArr)
# errList.append(errorRate)
return beta
def predict(beta, dataArr):
preArr = sigmoid(np.dot(dataArr, beta))
preArr[preArr > 0.5] = 1
preArr[preArr <= 0.5] = 0
return preArr
def cntErrRate(preLabel, label):
"""
calculate error rate
:param preLabel: predict label
:param label: real label
:return: error rate
"""
# print(f"preLabel = {preLabel}, label = {label}")
m = len(preLabel)
cnt = 0.0
for i in range(m):
if preLabel[i] != label[i]:
cnt += 1.0
return cnt / float(m)
def kFold(dataArr, labelArr, k, alpha, T):
"""
k-fold cross validation
:param dataArr:
:param labelArr:
:param k:
:param alpha: learning rate
:param T: iteration
:return: errList has k error rate
"""
m, n = dataArr.shape
index = np.arange(m)
np.random.shuffle(index)
dataArr = dataArr[index]
labelArr = labelArr[index]
errList = []
betaList = []
step = int(m / k)
# 一个数据分为10部分,让不整除的部分归在最后一个子集中。先计算前9部分。
for i in range(k - 1):
# get train data set and test data set
trainData = np.concatenate((dataArr[:i*step], dataArr[(i+1)*step:]))
trainLabel = np.concatenate((labelArr[:i*step], labelArr[(i+1)*step:]))
testData = dataArr[i*step:(i+1)*step]
testLabel = labelArr[i*step:(i+1)*step]
# get predict
beta = gradDescent(trainData, trainLabel, alpha, T)
pre = predict(beta, testData)
errList.append(cntErrRate(pre, testLabel))
betaList.append(beta)
# 计算最后一个区间为测试集的情况
beta = gradDescent(dataArr[:(k-1)*step],
labelArr[:(k-1)*step],
alpha, T)
pre = predict(beta, dataArr[(k-1)*step:])
errList.append(cntErrRate(pre, labelArr[(k-1)*step:]))
betaList.append(beta)
return betaList, errList
def LOO(dataArr, labelArr, alpha, T):
"""
Leave-One-Out
:param dataArr:
:param labelArr:
:param alpha:
:param T:
:return: errList
"""
m, n = dataArr.shape
errList = []
for i in range(m):
trainData = np.concatenate((dataArr[:i], dataArr[i+1:]))
trainLabel = np.concatenate((labelArr[:i], labelArr[i+1:]))
beta = gradDescent(trainData, trainLabel, alpha, T)
pre = predict(beta, dataArr[i])
errList.append(cntErrRate(pre, [labelArr[i]]))
return errList
def main():
dataArr, labelArr = getDataSet('dataR2.csv')
# test kFold()
k = 10
learningRate = 0.01
T = 1500
betaList, errList = kFold(dataArr, labelArr, k, learningRate, T)
averErr = sum(errList) / float(k)
print(f"{k} fold average error rate is {averErr}")
# 10次10折交叉验证结果
i = 10
kkErrList = []
while i:
i -= 1
_, errList = kFold(dataArr, labelArr, k, learningRate, T)
averageErr = sum(errList) / float(k)
kkErrList.append(averageErr)
kkAverErr = sum(kkErrList) / float(k)
print(f"{k}-th {k} fold average error rate is {kkAverErr}")
# test LOO()
errList = LOO(dataArr, labelArr, 0.01, 1500)
averErr = sum(errList) / float(len(errList))
print(f"LOO error rate is {averErr}")
if __name__ == '__main__':
main()
打印的结果为: