1. 参照图3.1,在2维空间中给出实例点,画出k为1和2时的k近邻法构成的空间划分(决策区域),并对其进行比较,体会k值选择与模型复杂度及预测准确率的关系。
import numpy as np
from sklearn.neighbors import KNeighborsClassifier
import matplotlib.pyplot as plt
from matplotlib.colors import ListedColormap
data = np.array([[5, 12, 1],
[6, 21, 0],
[14, 5, 0],
[16, 10, 0],
[13, 19, 0],
[13, 32, 1],
[17, 27, 1],
[18, 24, 1],
[20, 20, 0],
[23, 14, 1],
[23, 25, 1],
[23, 31, 1],
[26, 8, 0],
[30, 17, 1],
[30, 26, 1],
[34, 8, 0],
[34, 19, 1],
[37, 28, 1]])
X_train = data[:,0:2] #实例特征矩阵 每一行为一个实例的特征向量
y_train = data[:,-1] #类别标签
models = (KNeighborsClassifier(n_neighbors=1,n_jobs=-1),
KNeighborsClassifier(n_neighbors=2,n_jobs=-1),
)
models = (clf.fit(X_train,y_train) for clf in models)
titles = ('K Neighbors with k=1',
'K Neighbors with k=2')
fig = plt.figure(figsize=(15, 5))
plt.subplots_adjust(wspace=0.4, hspace=0.4)
x_min,x_max = X_train[:,0].min()-1,X_train[:,0].max()+1
y_min,y_max = X_train[:,1].min()-1,X_train[:,1].max()+1
#生成网格点
xx,yy = np.meshgrid(np.arange(x_min,x_max,0.2),np.arange(y_min,y_max,0.2))
for clf,title,ax in zip(models,titles,fig.subplots(1,2).flatten()):
Z = clf.predict(np.c_[xx.ravel(),yy.ravel()])
Z = Z.reshape(xx.shape)
colors = ('red', 'green', 'lightgreen', 'gray', 'cyan')
cmap = ListedColormap(colors[:len(np.unique(Z))])
ax.contourf(xx,yy,Z,cmap=cmap,alpha=0.5)
ax.scatter(X_train[:,0],X_train[:,1],c=y_train,s=50,edgecolors='k',cmap=cmap,alpha=0.5)
ax.set_title(title)
plt.show()
k值越小,模型越复杂(决策边界越复杂);k值越大,模型越简单(决策边界越简单)。
2. 利用例题3.2构造的kd树,求的最近邻点。
import numpy as np
from sklearn.neighbors import KDTree
train_data = np.array([[2,3],[5,4],[9,6],[4,7],[8,1],[7,2]])
tree = KDTree(train_data,leaf_size=2) #构造kd树,二叉树
#print(train_data[0]) #1维 [2,3]
#print(train_data[[0]]) #2维 [[2,3]]
dist,ind = tree.query(np.array([[3,4.5]]),k=1) #k=1最近邻
#x1 = train_data[ind[0]][0][0]
#x2 = train_data[ind[0]][0][1]
x1 = train_data[ind[0].squeeze()][0]
x2 = train_data[ind[0].squeeze()][1]
print("x点的最近邻点是({0}, {1})".format(x1, x2))
3. 基于kd树的最近邻搜索算法写出基于kd树的k近邻搜索算法。
输入:已构造的kd树;目标点x
输出:x的k近邻
1)在kd树中找出包含目标点x的叶结点:从根结点出发,递归向下访问树。若目标点x当前维的坐标小于切分点的坐标,则移动到左子结点,否则移动到右子结点,直到子结点为叶结点为止。
2)如果“当前k近邻点集”元素数量小于k或者叶节点(到x的)距离小于“当前k近邻点集”中最远点(到x的)距离,那么将叶结点插入“当前k近邻点集”。
3)递归地向上回退,在每个结点进行以下操作:
a)如果“当前k近邻点集”元素数量小于k或者当前节点(到x的)距离小于“当前k近邻点集”中最远点(到x的)距离,那么将该结点插入到“当前k近邻点集”。
b)检查另一子结点对应的区域是否与以目标点x为球心、以目标点x到“当前k近邻点集”中最远点的距离为半径的超球体相交。如果相交,可能在另一个子结点对应的区域内存在距目标点x更近的点,移动到另一子结点,接着,递归地进行最近邻搜索;如果不相交,向上回退。
4)当回退到根结点时,搜索结束,最后的“当前k近邻点集”即为x的k近邻点。
4. 思考k近邻算法的模型复杂度体现在哪里?什么情况下会造成过拟合?
k近邻法的复杂度体现在k值上。
k值比较小的时候容易造成过拟合。
k值比较大的时候容易造成欠拟合。
5. 给定一个二维空间的数据集T={正实例:(5,4),(9,6),(4,7);负实例:(2,3), (8,1),(7,2)},试基于欧氏距离,找到数据点S(5,3)的最近邻(k=1),并对S点进行分类预测。
1) 用“线性扫描”算法自编程实现。
import numpy as np
from collections import Counter
from draw import draw
class KNN:
def __init__(self, X_train, y_train, k=3):
# 所需参数初始化
self.k = k # 所取k值
self.X_train = X_train
self.y_train = y_train
def predict(self, X_new):
# 计算欧氏距离L2
dist_list = [(np.linalg.norm(X_new - self.X_train[i], ord=2), self.y_train[i])
for i in range(self.X_train.shape[0])]
# [(d0,-1),(d1,1)...]
# 对所有距离进行排序
dist_list.sort(key=lambda x: x[0])
# 取前k个最小距离对应的类别(也就是y值)
y_list = [dist_list[i][-1] for i in range(self.k)]
# [-1,1,1,-1...]
# 对上述k个点的分类进行统计
y_count = Counter(y_list).most_common()
# [(-1, 3), (1, 2)] #元组列表,(前面是类别,后面是出现频数)
return y_count[0][0]
def main():
# 训练数据
X_train = np.array([[5, 4],
[9, 6],
[4, 7],
[2, 3],
[8, 1],
[7, 2]])
y_train = np.array([1, 1, 1, -1, -1, -1])
# 测试数据
X_new = np.array([[5, 3]]) #用2维数组表示向量
# 绘图
draw(X_train, y_train, X_new)
# 不同的k(取奇数)对分类结果的影响
for k in range(1, 6, 2):
# 构建KNN实例
clf = KNN(X_train, y_train, k=k)
# 对测试数据进行分类预测
y_predict = clf.predict(X_new)
print("k={},被分类为:{}".format(k, y_predict))
if __name__ == "__main__":
main()
绘图函数:
def draw(X_train, y_train, X_new):
# 正负实例点初始化
X_po = np.zeros(X_train.shape[1])
X_ne = np.zeros(X_train.shape[1])
# 区分正、负实例点
for i in range(y_train.shape[0]):
if y_train[i] == 1:
X_po = np.vstack((X_po, X_train[i]))
else:
X_ne = np.vstack((X_ne, X_train[i]))
# 实例点绘图
plt.plot(X_po[1:, 0], X_po[1:, 1], "g*", label="1")
plt.plot(X_ne[1:, 0], X_ne[1:, 1], "rx", label="-1")
plt.plot(X_new[:, 0], X_new[:, 1], "bo", label="test_points")
# 测试点坐标值标注
for xy in zip(X_new[:, 0], X_new[:, 1]):
plt.annotate("test{}".format(xy), xy)
# 设置坐标轴
plt.axis([0, 10, 0, 10])
plt.xlabel("x1")
plt.ylabel("x2")
# 显示图例
plt.legend()
# 显示图像
plt.show()
前面的暴力求解代码存在一些问题,如:多数据处理(for循环,多进程);距离列表的排序,浪费时间和空间(可以使用堆这个数据结构)。
线性扫描的优化版本:
import numpy as np
from collections import Counter
from draw import draw
from concurrent import futures
import heapq #堆
import time
class KNN:
def __init__(self, X_train, y_train, k=3):
# 所需参数初始化
self.k = k # 所取k值
self.X_train = X_train
self.y_train = y_train
def predict_single(self, x_new):
# 计算与前k个样本点欧氏距离,距离取负值是把原问题转化为取前k个最大的距离
dist_list = [(-np.linalg.norm(x_new - self.X_train[i], ord=2), self.y_train[i], i)
for i in range(self.k)]
# 利用前k个距离构建堆
heapq.heapify(dist_list)
# 遍历计算与剩下样本点的欧式距离
for i in range(self.k, self.X_train.shape[0]):
dist_i = (-np.linalg.norm(x_new - self.X_train[i], ord=2), self.y_train[i], i)
# 若dist_i 比 dis_list的最小值大,则用dis_i替换该最小值,执行下堆操作
if dist_i[0] > dist_list[0][0]:
heapq.heappushpop(dist_list, dist_i)
# 若dist_i 比 dis_list的最小值小,堆保持不变,继续遍历
else:
continue
y_list = [dist_list[i][1] for i in range(self.k)]
# [-1,1,1,-1...]
# 对上述k个点的分类进行统计
y_count = Counter(y_list).most_common()
# {1:n,-1:m}
return y_count[0][0]
# 用多进程实现并行,处理多个值的搜索
def predict_many(self, X_new):
# 导入多进程
with futures.ThreadPoolExecutor(max_workers=4) as executor:
# 建立多进程任务
tasks = [executor.submit(self.predict_single, X_new[i]) for i in range(X_new.shape[0])]
# 驱动多进程运行
done_iter = futures.as_completed(tasks)
# 提取运行结果
res = [future.result() for future in done_iter]
return res
def main():
t0 = time.time()
# 训练数据
X_train = np.array([[5, 4],
[9, 6],
[4, 7],
[2, 3],
[8, 1],
[7, 2]])
y_train = np.array([1, 1, 1, -1, -1, -1])
# 测试数据
X_new = np.array([[5, 3], [9, 2]])
# 绘图
draw(X_train, y_train, X_new)
# 不同的k(取奇数)对分类结果的影响
for k in range(1, 6, 2):
# 构建KNN实例
clf = KNN(X_train, y_train, k=k)
# 对测试数据进行分类预测
y_predict = clf.predict_many(X_new)
print("k={},被分类为:{}".format(k, y_predict))
print("用时:{}s".format(round(time.time() - t0), 2))
if __name__ == "__main__":
main()
2)用"kd树"算法自编程实现。
# 构建kd树,搜索待预测点所属区域
from collections import namedtuple
import numpy as np
# 建立节点类
class Node(namedtuple("Node", "location left_child right_child")):
def __repr__(self):
return str(tuple(self))
# kd tree类
class KdTree():
def __init(self, k=1, p=2):
self.k = k
self.p = p
self.kdtree = None
# 构建kd tree
def _fit(self, X, depth=0):
try:
k = X.shape[1]
except IndexError as e:
return None
# 这里可以展开,通过方差选择axis
axis = depth % k
X = X[X[:, axis].argsort()]
median = X.shape[0] // 2
try:
X[median]
except IndexError:
return None
return Node(
location=X[median],
left_child=self._fit(X[:median], depth + 1),
right_child=self._fit(X[median + 1:], depth + 1)
)
def _search(self, point, tree=None, depth=0, best=None):
if tree is None:
return best
k = point.shape[1]
# 更新 branch
if point[0][depth % k] < tree.location[depth % k]:
next_branch = tree.left_child
else:
next_branch = tree.right_child
if not next_branch is None:
best = next_branch.location
return self._search(point, tree=next_branch, depth=depth + 1, best=best)
def fit(self, X):
self.kdtree = self._fit(X)
return self.kdtree
def predict(self, X):
res = self._search(X, self.kdtree)
return res
def main():
KNN = KdTree()
X_train = np.array([[5, 4],
[9, 6],
[4, 7],
[2, 3],
[8, 1],
[7, 2]])
KNN.fit(X_train) #构建kd树
X_new = np.array([[5, 3]])
res = KNN.predict(X_new) #搜索kd树
print(res)
if __name__ == "__main__":
main()
3)试调用sklearn.neighbors的KNeighborsClassifier模块,对S点进行分类预测,并对比近邻数k取值不同,对分类预测结果的影响。
KNeighborsClassifier模块重要的超参数:
其中weights参数的默认值为 "uniform",即每个训练实例权重相同,同等重要,k个近邻中多数的类别为预测类别,当选择"distance"时,离新实例点越近的训练实例权重越大;algorithm参数,程序会根据数据规模自动选择,当数据量比较小时,会用暴力求解,尽管你设置了kd/ball树,也是无效的,构造kd/ball树会有一定开销,数据量小时不适用,当实例维度d增加时,kd tree效率会下降,Ball tree可以解决高维查询问题,一般在20维以内,N>>2d时,适用kd tree;leaf_size参数在算法选择是kd树或ball树时,可以设置。
KNeighborsClassifier模块重要的方法:
import numpy as np
from sklearn.neighbors import KNeighborsClassifier
def main():
# 训练数据
X_train = np.array([[5, 4],
[9, 6],
[4, 7],
[2, 3],
[8, 1],
[7, 2]])
y_train = np.array([1, 1, 1, -1, -1, -1])
# 待预测数据
X_new = np.array([[5, 3]])
# 不同k值对结果的影响
for k in range(1, 6, 2):
# 构建实例
clf = KNeighborsClassifier(n_neighbors=k, n_jobs=-1)
# 选择合适算法
clf.fit(X_train, y_train)
print("邻近点:",clf.kneighbors(X_new)) #返回两个数组,一个是距离数组,另一个是索引数组
# 预测
y_predict = clf.predict(X_new)
print("属于每个类的概率",clf.predict_proba(X_new))
print("k={},被分类为:{}".format(k, y_predict), "预测正确率:{:.0%}".format(clf.score([[5, 3]], [[1]])))
if __name__ == "__main__":
main()
4)思考“线性扫描”算法和“kd树”算法的时间复杂度。
只考虑样本数N的影响:
线性扫描:计算N个距离 O(N)
kd tree:二叉树搜索 O(log2N)
实际上当维度d接近于N时,二者效率相当。