月亮数据预测(决策树和随机森林算法)

资料：https://download.csdn.net/download/m0_61504367/85399409

一、问题描述

月亮数据是sklearn工具库提供的一个数据集。它上用于分类和聚类算法的实践实验。图中每一个点是一条数据。其中(x1,x2)是特征组，颜色是标签值。如图所示。

采用决策树算法和随机森林算法进行月亮数据的标签预测，并给出预测的可视化结果。

二、实验目的

学习决策树算法和随机森林算法。

三、实验内容

3.1数据导入

from sklearn.datasets import make_moons
from sklearn.model_selection import train_test_split

3.2数据预处理

X, y = make_moons(n_samples=1000, noise=0.1)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=0)
plt.figure(0)
plt.axis([-1.5, 2.5, -0.75, 1.25])
plt.scatter(X_train[:, 0][y_train == 0], X_train[:, 1][y_train == 0], c='b', marker='o', s=10)
plt.scatter(X_train[:, 0][y_train == 1], X_train[:, 1][y_train == 1], c='r', marker='o', s=10)
plt.show()

3.3算法描述

决策树算法：
决策树（decision tree）是一个树结构（可以是二叉树或非二叉树）。其每个非叶节点表示一个特征属性上的测试，每个分支代表这个特征属性在某个值域上的输出，而每个叶节点存放一个类别。决策树学习的关键在于寻找最优划分特征和阈值。决策树回归算法用方差值来划分数据，最小化均方误差；决策树分类算法则用熵值来划分数据，最小化交叉熵。本实验是决策树分类问题，分值函数是使用交叉熵CE。
1.遍历所有可能的特征j与阈值
2.对每一组j和阈值, 由分值函数get_score计算score1和score2,
计算score。
3.选出score最小的一组j和阈值，样本的划分。

随机森林算法：
随机森林分类算法是一个包含多棵决策树的分类器。其输出的类别是由森林中的所有决策树输出类别的众数来确定的。
随机森林分类算法的基本思想依据：
1.虽然森林中的每一棵树都有一定的预测错误的概率，但是它们同时都出错的概率却是较低的。
2.在随机森林算法中，要求森林中的决策树相互独立。由于生成决策树的数据子集取自同一训练数据集，因此这些数据子集具有相同的分布。这使得所生成的决策树都比较相似。这就导致生成的决策树相互不独立。
3.增加森林中决策树之间完全相互独立的可能性的方法：随机森林算法在训练每一棵决策树时，要增加以下的额外步骤：在CART算法中作每一次数据划分时，随机地选取一部分特征进行遍历，而不是遍历全部特征。

3.4主要代码

moon.py

from sklearn.datasets import make_moons
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from lib.decision_tree_classifier import DecisionTreeClassifier
from lib.random_forest_classifier import RandomForestClassifier
from sklearn.metrics import accuracy_score
import numpy as np


def convert_to_vector(y):
    m = len(y)
    k = np.max(y) + 1
    v = np.zeros(m * k).reshape(m, k)
    for i in range(m):
        v[i][y[i]] = 1
    return v


X, y = make_moons(n_samples=1000, noise=0.1)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=0)
plt.figure(0)
plt.axis([-1.5, 2.5, -0.75, 1.25])
plt.scatter(X_train[:, 0][y_train == 0], X_train[:, 1][y_train == 0], c='b', marker='o', s=10)
plt.scatter(X_train[:, 0][y_train == 1], X_train[:, 1][y_train == 1], c='r', marker='o', s=10)
plt.show()

tree = DecisionTreeClassifier(max_depth=50)
tree.fit(X_train, convert_to_vector(y_train))
y_pred = tree.predict(X_test)
print("decision tree accuracy= {}".format(accuracy_score(y_test, y_pred)))

plt.figure(1)
x0s = np.linspace(-3, 4, 100)
x1s = np.linspace(-1, 6, 100)
x0, x1 = np.meshgrid(x0s, x1s)
W = np.c_[x0.ravel(), x1.ravel()]
u = tree.predict(W).reshape(x0.shape)
plt.axis([-1.5, 2.5, -0.75, 1.25])
plt.scatter(X_train[:, 0][y_train == 0], X_train[:, 1][y_train == 0], c='b', marker='o', s=30)
plt.scatter(X_train[:, 0][y_train == 1], X_train[:, 1][y_train == 1], c='g', marker='^', s=30)
plt.scatter(X_train[:, 0][y_train == 2], X_train[:, 1][y_train == 2], c='y', marker='s', s=30)
plt.contourf(x0, x1, u, c=u, alpha=0.2)
plt.show()
forest = RandomForestClassifier(max_depth=50, num_trees=100, feature_sample_rate=0.5, data_sample_rate=0.15)
forest.fit(X_train, convert_to_vector(y_train))
y_pred = forest.predict(X_test)
print("random forest accuracy= {}".format(accuracy_score(y_test, y_pred)))

plt.figure(2)
u = forest.predict(W).reshape(x0.shape)
plt.axis([-1.5, 2.5, -0.75, 1.25])
plt.scatter(X_train[:, 0][y_train == 0], X_train[:, 1][y_train == 0], c='b', marker='o', s=30)
plt.scatter(X_train[:, 0][y_train == 1], X_train[:, 1][y_train == 1], c='g', marker='^', s=30)
plt.scatter(X_train[:, 0][y_train == 2], X_train[:, 1][y_train == 2], c='y', marker='s', s=30)
plt.contourf(x0, x1, u, c=u, alpha=0.2)

plt.show()

决策树算法主要代码

import numpy as np
class Node:
    j = None
    theta = None
    p = None
    left = None
    right = None

class DecisionTreeBase:
    def __init__(self, max_depth, feature_sample_rate, get_score):
        self.max_depth = max_depth
        self.feature_sample_rate = feature_sample_rate
        self.get_score = get_score

    def split_data(self, j, theta, X, idx):
        idx1, idx2 = list(), list()
        for i in idx:
            value = X[i][j]
            if value <= theta:
                idx1.append(i)
            else:
                idx2.append(i)
        return idx1, idx2

    def get_random_features(self, n):
        shuffled = np.random.permutation(n)
        size = int(self.feature_sample_rate * n)
        selected = shuffled[:size]
        return selected

    def find_best_split(self, X, y, idx):
        m, n = X.shape
        best_score = float("inf")
        best_j = -1
        best_theta = float("inf")
        best_idx1, best_idx2 = list(), list()
        selected_j = self.get_random_features(n)
        for j in selected_j:
            thetas = set([x[j] for x in X])
            for theta in thetas:
                idx1, idx2 = self.split_data(j, theta, X, idx)
                if min(len(idx1), len(idx2)) == 0:
                    continue
                score1, score2 = self.get_score(y, idx1), self.get_score(y, idx2)
                w = 1.0 * len(idx1) / len(idx)
                score = w * score1 + (1 - w) * score2
                if score < best_score:
                    best_score = score
                    best_j = j
                    best_theta = theta
                    best_idx1 = idx1
                    best_idx2 = idx2
        return best_j, best_theta, best_idx1, best_idx2, best_score

    def generate_tree(self, X, y, idx, d):
        r = Node()
        r.p = np.average(y[idx], axis=0)
        if d == 0 or len(idx) < 2:
            return r
        current_score = self.get_score(y, idx)
        j, theta, idx1, idx2, score = self.find_best_split(X, y, idx)
        if score >= current_score:
            return r
        r.j = j
        r.theta = theta
        r.left = self.generate_tree(X, y, idx1, d - 1)
        r.right = self.generate_tree(X, y, idx2, d - 1)
        return r

    def fit(self, X, y):
        self.root = self.generate_tree(X, y, range(len(X)), self.max_depth)

    def get_prediction(self, r, x):
        if r.left == None and r.right == None:
            return r.p
        value = x[r.j]
        if value <= r.theta:
            return self.get_prediction(r.left, x)
        else:
            return self.get_prediction(r.right, x)

    def predict(self, X):
        y = list()
        for i in range(len(X)):
            y.append(self.get_prediction(self.root, X[i]))
        return np.array(y)




def get_impurity(y, idx):
    p = np.average(y[idx], axis=0)  
    return 1 - p.dot(p.T)

def get_entropy(y, idx):
    _, k = y.shape
    p = np.average(y[idx], axis=0) 
    return - np.log(p + 0.001 * np.random.rand(k)).dot(p.T)
    
class DecisionTreeClassifier(DecisionTreeBase):
    def __init__(self, max_depth=0, feature_sample_rate=1.0):
        super().__init__(max_depth = max_depth, 
            feature_sample_rate = feature_sample_rate,
            get_score = get_entropy)
    
    def predict_proba(self, X):
        return super().predict(X)
    
    def predict(self, X):
        proba = self.predict_proba(X)
        return np.argmax(proba,axis=1)

随机森林算法主要代码

import numpy as np
class Node:
    j = None
    theta = None
    p = None
    left = None
    right = None


class DecisionTreeBase:
    def __init__(self, max_depth, feature_sample_rate, get_score):
        self.max_depth = max_depth
        self.feature_sample_rate = feature_sample_rate
        self.get_score = get_score

    def split_data(self, j, theta, X, idx):
        idx1, idx2 = list(), list()
        for i in idx:
            value = X[i][j]
            if value <= theta:
                idx1.append(i)
            else:
                idx2.append(i)
        return idx1, idx2

    def get_random_features(self, n):
        shuffled = np.random.permutation(n)
        size = int(self.feature_sample_rate * n)
        selected = shuffled[:size]
        return selected

    def find_best_split(self, X, y, idx):
        m, n = X.shape
        best_score = float("inf")
        best_j = -1
        best_theta = float("inf")
        best_idx1, best_idx2 = list(), list()
        selected_j = self.get_random_features(n)
        for j in selected_j:
            thetas = set([x[j] for x in X])
            for theta in thetas:
                idx1, idx2 = self.split_data(j, theta, X, idx)
                if min(len(idx1), len(idx2)) == 0:
                    continue
                score1, score2 = self.get_score(y, idx1), self.get_score(y, idx2)
                w = 1.0 * len(idx1) / len(idx)
                score = w * score1 + (1 - w) * score2
                if score < best_score:
                    best_score = score
                    best_j = j
                    best_theta = theta
                    best_idx1 = idx1
                    best_idx2 = idx2
        return best_j, best_theta, best_idx1, best_idx2, best_score

    def generate_tree(self, X, y, idx, d):
        r = Node()
        r.p = np.average(y[idx], axis=0)
        if d == 0 or len(idx) < 2:
            return r
        current_score = self.get_score(y, idx)
        j, theta, idx1, idx2, score = self.find_best_split(X, y, idx)
        if score >= current_score:
            return r
        r.j = j
        r.theta = theta
        r.left = self.generate_tree(X, y, idx1, d - 1)
        r.right = self.generate_tree(X, y, idx2, d - 1)
        return r

    def fit(self, X, y):
        self.root = self.generate_tree(X, y, range(len(X)), self.max_depth)

    def get_prediction(self, r, x):
        if r.left == None and r.right == None:
            return r.p
        value = x[r.j]
        if value <= r.theta:
            return self.get_prediction(r.left, x)
        else:
            return self.get_prediction(r.right, x)

    def predict(self, X):
        y = list()
        for i in range(len(X)):
            y.append(self.get_prediction(self.root, X[i]))
        return np.array(y)


def get_impurity(y, idx):
    p = np.average(y[idx], axis=0)
    return 1 - p.dot(p.T)


def get_entropy(y, idx):
    _, k = y.shape
    p = np.average(y[idx], axis=0)
    return - np.log(p + 0.001 * np.random.rand(k)).dot(p.T)


class DecisionTreeClassifier(DecisionTreeBase):
    def __init__(self, max_depth=0, feature_sample_rate=1.0):
        super().__init__(max_depth=max_depth,
                         feature_sample_rate=feature_sample_rate,
                         get_score=get_entropy)

    def predict_proba(self, X):
        return super().predict(X)

    def predict(self, X):
        proba = self.predict_proba(X)
        return np.argmax(proba, axis=1)


def get_impurity(y, idx):
    p = np.average(y[idx], axis=0)
    return 1 - p.dot(p.T)


def get_entropy(y, idx):
    _, k = y.shape
    p = np.average(y[idx], axis=0)
    return - np.log(p + 0.001 * np.random.rand(k)).dot(p.T)


class DecisionTreeClassifier(DecisionTreeBase):
    def __init__(self, max_depth=0, feature_sample_rate=1.0):
        super().__init__(max_depth=max_depth,
                         feature_sample_rate=feature_sample_rate,
                         get_score=get_entropy)

    def predict_proba(self, X):
        return super().predict(X)

    def predict(self, X):
        proba = self.predict_proba(X)
        return np.argmax(proba, axis=1)

class RandomForestClassifier:        
    def __init__(self, num_trees,max_depth, feature_sample_rate, 
            data_sample_rate, random_state = 0):
        self.max_depth, self.num_trees = max_depth, num_trees
        self.feature_sample_rate = feature_sample_rate
        self.data_sample_rate = data_sample_rate
        self.trees = []
        np.random.seed(random_state)
    
    def get_data_samples(self, X, y):
        shuffled_indices = np.random.permutation(len(X))
        size = int(self.data_sample_rate * len(X))
        selected_indices = shuffled_indices[:size]
        return X[selected_indices], y[selected_indices]
    
    def fit(self, X, y):
        for t in range(self.num_trees):
            X_t, y_t = self.get_data_samples(X, y)
            model = DecisionTreeClassifier(
                max_depth = self.max_depth, 
                feature_sample_rate = self.feature_sample_rate)
            model.fit(X_t, y_t)
            self.trees.append(model)
    
    def predict_proba(self, X):
        probas = np.array([tree.predict_proba(X) for tree in self.trees])
        return np.average(probas, axis=0)
        
    def predict(self, X):
        proba = self.predict_proba(X)
        return np.argmax(proba, axis=1)   
    

四、实验结果及分析

原始数据：

决策树预测结果
（1）max_depath=5

（2）max_depath=3

（3）max_depath=50

随机森林预测结果
（1）max_depath=5，num_tree =100

（2）max_depath=3，num_tree =100

（3）max_depath=50，num_tree =100

五、获取资源

https://download.csdn.net/download/m0_61504367/85399409