前情回顾
结论速递
虽然本次助教提供了参考代码,但还是尝试基于自己的认知尝试从头实现了一个CART树。
本次任务自己编程实现了一棵CART树,实现了分类及回归任务,可以实现最大树深度限制。
通过这个任务,对CART树的原理有了进一步的理解。
代码实现
本次实现的CART树可以完成分类及回归任务,可以实现最大树深度限制(最大叶结点树暂时还没有实现),连续属性离散化。
基础库导入及函数定义
定义了gini系数的计算,以及MSE,MAE信息增益的计算
import numpy as np
def _labelcount(data):
mask = np.unique(data)
tmp = []
for each in mask:
tmp.append(np.sum(data == each))
p = np.array(tmp) / len(data)
return mask, p
def _gini(data):
_, dataK = _labelcount(data)
gini = 1 - (dataK**2).sum()
return gini
def _MSE(y, yP):
return ((y-yP)**2).sum()
def _MAE(y, yP):
return (np.absolute(y-yP)).sum()
def G_classification(x, y):
giniY = _gini(y)
mask = np.unique(x)
tmpGini = []
count = []
for each in mask:
tmpGini.append(_gini(y[x == each]))
count.append(np.sum(x == each))
giniYX = (np.array(tmpGini)*np.array(count)/len(x)).sum()
return giniY - giniYX
def G_regression(x, y, yNow = None, function = "MSE"):
if yNow is None:
yNow = y.mean()
if function == "MSE":
f = _MSE
elif function == "MAE":
f = _MAE
total = f(y,yNow)/len(y)
yLeft = y[x==0].mean()
left = f(y[x==0],yLeft)/len(y)
yRight = y[x==1].mean()
right = f(y[x==1],yRight)/len(y)
return total - left - right, yLeft, yRight
树节点类定义
用于存储信息
class TreeNode():
def __init__(self, leaf = False, nodeLabel = None, depth = 0, father = None):
self.leaf = leaf
#self.label = nodeLabel
#self.feature = None #idx
#self.divide = None #idx tuple
#self.left = None
#self.right = None
self.father = father
self.depth = depth
核心树实现
class Tree():
def __init__(self, max_depth = None, min_node_count = 1, max_leaf_nodes = None,
random_state = 33, splitter = "random",
regression = False, metric_type = "MSE"):
self.type = "regression" if regression else "classification"
if regression:
self.metric_type = metric_type
self.decision_metric = G_regression
else:
self.decision_metric = G_classification
self.max_depth = max_depth
self.min_node_count = min_node_count
self.max_leaf_nodes = max_leaf_nodes
self.random_state = random_state
self.splitter = splitter
def fit(self,x,y):
self.root = TreeNode(leaf = False, depth = 0)
self.leaf_node_count = 0
self._growth(x,y,self.root)
def predict(self,x):
return self._search_prediction(x,self.root)
def _divide_node(self,x):
#print(x)
if len(np.unique(x)) <= len(x)/2:
#discrete
choiceLis = np.unique(x)
divideLis = [(each) for each in choiceLis]
#divideMotherLis = [each for each in permutations(choiceLis)]
#divideLis = []
#for num in range(1,int(np.ceil(len(choiceLis)/2))+1):
# divideLis.extend([tuple(sorted(list(each[:num])))
# for each in divideMotherLis])
divideLis = list(set(divideLis))
return divideLis
else:
if self.splitter == "random":
count = max(min(int(np.ceil(len(x)/self.min_node_count)),10),2)
np.random.seed = self.random_state
countAct = np.random.randint(2,count+1)
minVal = x.min()
maxVal = x.max()
divideLis = list(np.linspace(minVal,maxVal,countAct))
else:
divideLis = list(x)
return divideLis
def _growth(self,x,y,node):
#deep first
if ((node.depth == self.max_depth)
or (len(y) <= self.min_node_count)
or (len(np.unique(y)) == 1)):
node.leaf = True
self.leaf_node_count += 1
if self.type == "regression":
node.label = y.mean()
else:
node.label = np.argmax(np.bincount(y.flatten()))
return None
feature, divide = self._choose_best_split(x,y)
node.feature = feature
node.divide = divide
if type(divide) == tuple:
idx = [each in divide for each in x[:,feature]]
else:
idx = [each < divide for each in x[:,feature]]
node.left = TreeNode(leaf = False, father = node, depth = node.depth+1)
#print(x[idx,:])
_ = self._growth(x[idx,:],y[idx],node.left)
idx = [not each for each in idx]
node.right = TreeNode(leaf = False, father = node, depth = node.depth+1)
_ = self._growth(x[idx,:],y[idx],node.right)
def _choose_best_split(self,x,y):
divideLis = []
featureLis = []
for eachfeature in range(x.shape[1]):
lis = self._divide_node(x[:,eachfeature])
divideLis.extend(lis)
featureLis.extend([eachfeature]*len(lis))
maxind = 0
maxVal = 0
for i in range(len(divideLis)):
divide = divideLis[i]
feature = featureLis[i]
try:
xD = [0 if each in divide else 1 for each in x[:,feature]]
except:
xD = [0 if each < divide else 1 for each in x[:,feature]]
if self.type == "classification":
new = self.decision_metric(xD, y)
elif self.type == "regression":
yNow = y.mean()
new,_,_ = self.decision_metric(xD, y, yNow, function = self.metric_type)
maxind = i if new > maxVal else maxind
index = maxind
divide = divideLis[index]
feature = featureLis[index]
return feature, divide
def _search_prediction(self,x,node):
if node.leaf == True:
return node.label
else:
if type(node.divide) == tuple:
bo = x[node.feature] in node.divide
else:
bo = x[node.feature] < node.divide
if bo:
return(self._search_prediction(x,node.left))
else:
return(self._search_prediction(x,node.right))
封装
class ClassificationTree():
def __init__(self, max_depth = None, min_node_count = 1, max_leaf_nodes = None,
random_state = 33, splitter = "random",):
self.tree = Tree(max_depth = max_depth,
min_node_count = min_node_count,
max_leaf_nodes = max_leaf_nodes,
random_state = random_state,
splitter = splitter,
regression = False)
def fit(self,x,y):
self.tree.fit(x,y)
def predict(self,x):
y = []
for i in range(x.shape[0]):
y.append(self.tree.predict(x[i,:]))
return np.array(y)#[np.newaxis,:]
class RegressionTree():
def __init__(self, max_depth = None, min_node_count = 1, max_leaf_nodes = None,
random_state = 33, splitter = "random", metric_type = "MSE"):
self.tree = Tree(max_depth = max_depth,
min_node_count = min_node_count,
max_leaf_nodes = max_leaf_nodes,
random_state = random_state,
splitter = splitter,
regression = True,
metric_type = metric_type)
def fit(self,x,y):
self.tree.fit(x,y)
def predict(self,x):
y = []
for i in range(x.shape[0]):
y.append(self.tree.predict(x[i,:]))
return np.array(y)[np.newaxis,:]
测试
在Iris数据集上进行了分类训练测试,与sklearn中的决策树效果一致。
在玩具数据集上进行了回归训练测试,与sklearn中的差距较大,待进一步解决。