Numpy实现DecisionTree

一、Python所有方向的学习路线

Python所有方向路线就是把Python常用的技术点做整理，形成各个领域的知识点汇总，它的用处就在于，你可以按照上面的知识点去找对应的学习资源，保证自己学得较为全面。

二、学习软件

工欲善其事必先利其器。学习Python常用的开发软件都在这里了，给大家节省了很多时间。

三、入门学习视频

我们在看视频学习的时候，不能光动眼动脑不动手，比较科学的学习方法是在理解之后运用它们，这时候练手项目就很适合了。

网上学习资料一大堆，但如果学到的知识不成体系，遇到问题时只是浅尝辄止，不再深入研究，那么很难做到真正的技术提升。

需要这份系统化学习资料的朋友，可以戳这里获取

一个人可以走的很快，但一群人才能走的更远！不论你是正从事IT行业的老鸟或是对IT行业感兴趣的新人，都欢迎加入我们的的圈子（技术交流、学习资源、职场吐槽、大厂内推、面试辅导），让我们一起学习成长！

---

min_samples_split: int

The minimum number of samples needed to make a split when building a tree.

min_impurity: float

The minimum impurity required to split the tree further.

max_depth: int

The maximum depth of a tree.

loss: function

Loss function that is used for Gradient Boosting models to calculate impurity.

“”"

def init(self, min_samples_split=2, min_impurity=1e-7,

max_depth=float(“inf”), loss=None):

self.root = None # Root node in dec. tree

Minimum n of samples to justify split

self.min_samples_split = min_samples_split

The minimum impurity to justify split

self.min_impurity = min_impurity

The maximum depth to grow the tree to

self.max_depth = max_depth

Function to calculate impurity (classif.=>info gain, regr=>variance reduct.)

self._impurity_calculation = None

Function to determine prediction of y at leaf

self._leaf_value_calculation = None

If y is one-hot encoded (multi-dim) or not (one-dim)

self.one_dim = None

If Gradient Boost

self.loss = loss

def fit(self, X, y, loss=None):

“”" Build decision tree “”"

self.one_dim = len(np.shape(y)) == 1

self.root = self._build_tree(X, y)

self.loss=None

def _build_tree(self, X, y, current_depth=0):

“”" Recursive method which builds out the decision tree and splits X and respective y

on the feature of X which (based on impurity) best separates the data"“”

largest_impurity = 0

best_criteria = None # Feature index and threshold

best_sets = None # Subsets of the data

Check if expansion of y is needed

if len(np.shape(y)) == 1:

y = np.expand_dims(y, axis=1)

Add y as last column of X

Xy = np.concatenate((X, y), axis=1)

n_samples, n_features = np.shape(X)

if n_samples >= self.min_samples_split and current_depth <= self.max_depth:

Calculate the impurity for each feature

for feature_i in range(n_features):

All values of feature_i

feature_values = np.expand_dims(X[:, feature_i], axis=1)

unique_values = np.unique(feature_values)

Iterate through all unique values of feature column i and

calculate the impurity

for threshold in unique_values:

Divide X and y depending on if the feature value of X at index feature_i

meets the threshold

Xy1, Xy2 = divide_on_feature(Xy, feature_i, threshold)

if len(Xy1) > 0 and len(Xy2) > 0:

Select the y-values of the two sets

y1 = Xy1[:, n_features:]

y2 = Xy2[:, n_features:]

Calculate impurity

impurity = self._impurity_calculation(y, y1, y2)

If this threshold resulted in a higher information gain than previously

recorded save the threshold value and the feature

index

if impurity > largest_impurity:

largest_impurity = impurity

best_criteria = {“feature_i”: feature_i, “threshold”: threshold}

best_sets = {

“leftX”: Xy1[:, :n_features], # X of left subtree

“lefty”: Xy1[:, n_features:], # y of left subtree

“rightX”: Xy2[:, :n_features], # X of right subtree

“righty”: Xy2[:, n_features:] # y of right subtree

}

if largest_impurity > self.min_impurity:

Build subtrees for the right and left branches

true_branch = self._build_tree(best_sets[“leftX”], best_sets[“lefty”], current_depth + 1)

false_branch = self._build_tree(best_sets[“rightX”], best_sets[“righty”], current_depth + 1)

return DecisionNode(feature_i=best_criteria[“feature_i”], threshold=best_criteria[

“threshold”], true_branch=true_branch, false_branch=false_branch)

We’re at leaf => determine value

leaf_value = self._leaf_value_calculation(y)

return DecisionNode(value=leaf_value)

def predict_value(self, x, tree=None):

“”" Do a recursive search down the tree and make a prediction of the data sample by the

value of the leaf that we end up at “”"

if tree is None:

tree = self.root

If we have a value (i.e we’re at a leaf) => return value as the prediction

if tree.value is not None:

return tree.value

Choose the feature that we will test

feature_value = x[tree.feature_i]

Determine if we will follow left or right branch

branch = tree.false_branch

if isinstance(feature_value, int) or isinstance(feature_value, float):

if feature_value >= tree.threshold:

branch = tree.true_branch

elif feature_value == tree.threshold:

branch = tree.true_branch

Test subtree

return self.predict_value(x, branch)

def predict(self, X):

“”" Classify samples one by one and return the set of labels “”"

y_pred = [self.predict_value(sample) for sample in X]

return y_pred

def print_tree(self, tree=None, indent=" "):

“”" Recursively print the decision tree “”"

if not tree:

tree = self.root

If we’re at leaf => print the label

if tree.value is not None:

print (tree.value)

Go deeper down the tree

else:

Print test

print ("%s:%s? " % (tree.feature_i, tree.threshold))

Print the true scenario

print (“%sT->” % (indent), end=“”)

self.print_tree(tree.true_branch, indent + indent)

Print the false scenario

print (“%sF->” % (indent), end=“”)

self.print_tree(tree.false_branch, indent + indent)

class XGBoostRegressionTree(DecisionTree):

“”"

Regression tree for XGBoost

• Reference -

http://xgboost.readthedocs.io/en/latest/model.html

“”"

def _split(self, y):

“”" y contains y_true in left half of the middle column and

y_pred in the right half. Split and return the two matrices “”"

col = int(np.shape(y)[1]/2)

y, y_pred = y[:, :col], y[:, col:]

return y, y_pred

def _gain(self, y, y_pred):

nominator = np.power((y * self.loss.gradient(y, y_pred)).sum(), 2)

denominator = self.loss.hess(y, y_pred).sum()

return 0.5 * (nominator / denominator)

def _gain_by_taylor(self, y, y1, y2):

Split

y, y_pred = self._split(y)

y1, y1_pred = self._split(y1)

y2, y2_pred = self._split(y2)

true_gain = self._gain(y1, y1_pred)

false_gain = self._gain(y2, y2_pred)

gain = self._gain(y, y_pred)

return true_gain + false_gain - gain

def _approximate_update(self, y):

y split into y, y_pred

y, y_pred = self._split(y)

Newton’s Method

gradient = np.sum(y * self.loss.gradient(y, y_pred), axis=0)

hessian = np.sum(self.loss.hess(y, y_pred), axis=0)

update_approximation = gradient / hessian

return update_approximation

def fit(self, X, y):

self._impurity_calculation = self._gain_by_taylor

self._leaf_value_calculation = self._approximate_update

super(XGBoostRegressionTree, self).fit(X, y)

class RegressionTree(DecisionTree):

（1）Python所有方向的学习路线（新版）

这是我花了几天的时间去把Python所有方向的技术点做的整理，形成各个领域的知识点汇总，它的用处就在于，你可以按照上面的知识点去找对应的学习资源，保证自己学得较为全面。

最近我才对这些路线做了一下新的更新，知识体系更全面了。

（2）Python学习视频

包含了Python入门、爬虫、数据分析和web开发的学习视频，总共100多个，虽然没有那么全面，但是对于入门来说是没问题的，学完这些之后，你可以按照我上面的学习路线去网上找其他的知识资源进行进阶。

（3）100多个练手项目

我们在看视频学习的时候，不能光动眼动脑不动手，比较科学的学习方法是在理解之后运用它们，这时候练手项目就很适合了，只是里面的项目比较多，水平也是参差不齐，大家可以挑自己能做的项目去练练。

网上学习资料一大堆，但如果学到的知识不成体系，遇到问题时只是浅尝辄止，不再深入研究，那么很难做到真正的技术提升。

需要这份系统化学习资料的朋友，可以戳这里获取

一个人可以走的很快，但一群人才能走的更远！不论你是正从事IT行业的老鸟或是对IT行业感兴趣的新人，都欢迎加入我们的的圈子（技术交流、学习资源、职场吐槽、大厂内推、面试辅导），让我们一起学习成长！