ID3决策树(Python实现)

from math import log

#计算香农熵(信息熵)
def calcShannonEnt(dataSet):
    numEntries = len(dataSet)
    labelCounts = {}
    for featVec in dataSet:
        currentLabel = featVec[-1]
        if currentLabel not in labelCounts.keys():
            labelCounts[currentLabel] = 0
        labelCounts[currentLabel] += 1
        shannonEnt = 0.0
        for key in labelCounts:
            prob = float(labelCounts[key]) / numEntries
            shannonEnt -= prob*log(prob,2)
    return shannonEnt

# 创造数据集
def creatDataSet():
    dataSet = [[1, 1, 'yes'],
               [1, 1, 'yes'],
               [1, 0, 'no'],
               [0, 1, 'no'],
               [0, 1, 'no']]
    labels = ['no surfacing', 'flippers']
    return dataSet, labels


#划分数据集
def splitDataSet(dataSet, axis, value):
    retDataSet = []
    for featVec in dataSet:
        if featVec[axis] == value:
            reducedFeatVec = featVec[:axis]
            reducedFeatVec.extend(featVec[axis+1:])
            retDataSet.append(reducedFeatVec)
    return retDataSet

#选择最佳的数据划分方式
def chooseBestFeatureToSplit(dataSet):
    numFeatures = len(dataSet[0]) - 1
    baseEntropy = calcShannonEnt(dataSet)
    bestInfoGain = 0.0
    bestFeature = -1
    for i in range(numFeatures):
        featList = [example[i]  for example in dataSet]
        uniqueVals = set(featList)
        newEntropy = 0.0
        for value in uniqueVals:
            subDataSet = splitDataSet(dataSet, i, value)
            prob = len(subDataSet) / float(len(dataSet))
            newEntropy += prob * calShannonEnt(subDataSet)
        infoGain = baseEntropy - newEntropy
        if infoGain > bestInfoGain:
            bestInfoGain= infoGain
            bestFeature = i
    return bestFeature

def majorityCnt(classList):
    classCount = {}
    for vote in classList:
        if vote not in classCount.keys():
            classCount[vote] = 0
            classCount[vote] += 1
        sortedClassCount = sorted(classCount.iteritems(), key = operator.itemgetter(1), reverse=True)
    return sortedClassCount[0][0]

def createTree(dataSet, labels):
    classList = [example[-1] for example in dataSet]
    if classList.count(classList[0]) == len(classList):
        return classList[0]
    if len(dataSet[0]) == 1:
        return majorityCnt(classList)
    bestFeat = chooseBestFeatureToSplit(dataSet)
    bestFeatLabel = labels[bestFeat]
    myTree = {bestFeatLabel:{}}
    del (labels[bestFeat])
    featValues = [example[bestFeat] for example in dataSet]
    uniqueValues = set(featValues)
    for value in uniqueValues:
        subLabels = labels[:]
        myTree[bestFeatLabel][value] = createTree(splitDataSet(dataSet,bestFeat, value), subLabels)
    return myTree

参考:Peter Harrington. 机器学习实战[M]. 2013.

ID3决策树是一种经典的机器学习算法,用于分类问题。下面是ID3决策树Python实现的基本步骤: 1. 数据准备:首先,需要准备一个包含特征和标签的数据集。特征可以是离散值或连续值,标签是分类的结果。 2. 计算信息增益:使用信息增益来选择最佳的特征作为当前节点的划分依据。信息增益表示在已知某个特征的情况下,对分类结果的不确定性减少程度。 3. 构建决策树:根据信息增益选择最佳特征后,将该特征作为当前节点的划分依据,并根据该特征的取值创建子节点。递归地对每个子节点进行上述步骤,直到满足终止条件(如所有样本属于同一类别或没有更多特征可供选择)。 4. 预测:使用构建好的决策树对新样本进行分类预测。 下面是一个简单的ID3决策树Python实现示例: ```python import numpy as np def calc_entropy(labels): # 计算标签的熵 unique_labels, counts = np.unique(labels, return_counts=True) probabilities = counts / len(labels) entropy = -np.sum(probabilities * np.log2(probabilities)) return entropy def calc_information_gain(data, labels, feature_index): # 计算特征的信息增益 feature_values = np.unique(data[:, feature_index]) entropy = calc_entropy(labels) new_entropy = 0 for value in feature_values: subset_labels = labels[data[:, feature_index] == value] subset_entropy = calc_entropy(subset_labels) subset_probability = len(subset_labels) / len(labels) new_entropy += subset_probability * subset_entropy information_gain = entropy - new_entropy return information_gain def create_decision_tree(data, labels, features): # 创建决策树 if len(np.unique(labels)) == 1: # 如果所有样本属于同一类别,则返回该类别作为叶节点 return labels if len(features) == 0: # 如果没有更多特征可供选择,则返回样本中最多的类别作为叶节点 unique_labels, counts = np.unique(labels, return_counts=True) return unique_labels[np.argmax(counts)] best_feature_index = np.argmax([calc_information_gain(data, labels, i) for i in range(len(features))]) best_feature = features[best_feature_index] decision_tree = {best_feature: {}} feature_values = np.unique(data[:, best_feature_index]) for value in feature_values: subset_data = data[data[:, best_feature_index] == value] subset_labels = labels[data[:, best_feature_index] == value] subset_features = features.copy() subset_features.remove(best_feature) decision_tree[best_feature][value] = create_decision_tree(subset_data, subset_labels, subset_features) return decision_tree def predict(decision_tree, sample): # 使用决策树进行预测 if isinstance(decision_tree, str): return decision_tree feature = list(decision_tree.keys()) value = sample[feature] subtree = decision_tree[feature][value] return predict(subtree, sample) # 示例用法 data = np.array([[1, 'S', 'M'], [1, 'M', 'M'], [1, 'M', 'L'], [1, 'S', 'L'], [1, 'S', 'M'], [2, 'S', 'M'], [2, 'M', 'M'], [2, 'M', 'L'], [2, 'L', 'L'], [2, 'L', 'L'], [3, 'L', 'L'], [3, 'M', 'L'], [3, 'M', 'M'], [3, 'L', 'M'], [3, 'L', 'L']]) labels = np.array(['N', 'N', 'Y', 'Y', 'N', 'N', 'N', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'N']) features = ['age', 'income', 'student'] decision_tree = create_decision_tree(data, labels, features) sample = {'age': 2, 'income': 'M', 'student': 'M'} prediction = predict(decision_tree, sample) print("预测结果:", prediction) ```
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值