《机器学习实战》个人学习记录笔记(五)———决策树构建与分类

第三章 决策树

PS:个人笔记 根据《机器学习实战》这本书,Jack-Cui的博客,以及深度眸的视频进行学习

1 决策树构建

ID3算法的核心是在决策树各个结点上对应信息增益准则选择特征,递归地构建决策树。具体方法是:从根结点(root node)开始,对结点计算所有可能的特征的信息增益,选择信息增益最大的特征作为结点的特征,由该特征的不同取值建立子节点;再对子结点递归地调用以上方法,构建决策树;直到所有特征的信息增益均很小或没有特征可以选择为止,最后得到一个决策树。ID3相当于用极大似然法进行概率模型的选择。


2 编写代码构建决策树

from math import log
import operator

def calcShannonEnt(dataSet):
    numEntires = len(dataSet)                     
    labelCounts = {}                                
    for featVec in dataSet:                      
        currentLabel = featVec[-1]                
        if currentLabel not in labelCounts.keys():  
            labelCounts[currentLabel] = 0
        labelCounts[currentLabel] += 1             
    shannonEnt = 0.0                               
    for key in labelCounts:                      
        prob = float(labelCounts[key]) / numEntires    
        shannonEnt -= prob * log(prob, 2)           
    return shannonEnt                              

def createDataSet():
    dataSet = [[0, 0, 0, 0, 'no'],                       
            [0, 0, 0, 1, 'no'],
            [0, 1, 0, 1, 'yes'],
            [0, 1, 1, 0, 'yes'],
            [0, 0, 0, 0, 'no'],
            [1, 0, 0, 0, 'no'],
            [1, 0, 0, 1, 'no'],
            [1, 1, 1, 1, 'yes'],
            [1, 0, 1, 2, 'yes'],
            [1, 0, 1, 2, 'yes'],
            [2, 0, 1, 2, 'yes'],
            [2, 0, 1, 1, 'yes'],
            [2, 1, 0, 1, 'yes'],
            [2, 1, 0, 2, 'yes'],
            [2, 0, 0, 0, 'no']]
    labels = ['年龄', '有工作', '有自己的房子', '信贷情况']      
    return dataSet, labels                          

def splitDataSet(dataSet, axis, value):       
    retDataSet = []                                        
    for featVec in dataSet:                           
        if featVec[axis] == value:
            reducedFeatVec = featVec[:axis]                
            reducedFeatVec.extend(featVec[axis+1:])     
            retDataSet.append(reducedFeatVec)
    return retDataSet                                     

def chooseBestFeatureToSplit(dataSet):
    numFeatures = len(dataSet[0]) - 1                   
    baseEntropy = calcShannonEnt(dataSet)              
    bestInfoGain = 0.0                                 
    bestFeature = -1                                    
    for i in range(numFeatures):                       
        featList = [example[i] for example in dataSet]
        uniqueVals = set(featList)                       
        newEntropy = 0.0                                 
        for value in uniqueVals:                       
            subDataSet = splitDataSet(dataSet, i, value)      
            prob = len(subDataSet) / float(len(dataSet))         
            newEntropy += prob * calcShannonEnt(subDataSet)    
        infoGain = baseEntropy - newEntropy                   
        # print("第%d个特征的增益为%.3f" % (i, infoGain))         
        if (infoGain > bestInfoGain):                         
            bestInfoGain = infoGain                       
            bestFeature = i                               
    return bestFeature                                     

"""
函数说明:统计classList中出现此处最多的元素(类标签)

Parameters:
    classList - 类标签列表
Returns:
    sortedClassCount[0][0] - 出现此处最多的元素(类标签)
"""
def majorityCnt(classList):
    classCount = {}
    for vote in classList:                                        #统计classList中每个元素出现的次数
        if vote not in classCount.keys():classCount[vote] = 0   
        classCount[vote] += 1
    sortedClassCount = sorted(classCount.items(), key = operator.itemgetter(1), reverse = True)        #根据字典的值降序排序
    return sortedClassCount[0][0]                                #返回classList中出现次数最多的元素

"""
函数说明:创建决策树

Parameters:
    dataSet - 训练数据集
    labels - 分类属性标签
    featLabels - 存储选择的最优特征标签
Returns:
    myTree - 决策树
Author:
    Jack Cui
Blog:
    http://blog.csdn.net/c406495762
Modify:
    2017-07-25
"""
def createTree(dataSet, labels, featLabels):
    classList = [example[-1] for example in dataSet]            #取分类标签(是否放贷:yes or no)
    if classList.count(classList[0]) == len(classList):            #如果类别完全相同则停止继续划分;count()计算一个类别的个数=类别列表里类别数
        return classList[0]
    if len(dataSet[0]) == 1:                                    #遍历完所有特征时返回出现次数最多的类标签;没有特征时,用类别投票表决处理
        return majorityCnt(classList)
    bestFeat = chooseBestFeatureToSplit(dataSet)                #选择最优特征 ;
    bestFeatLabel = labels[bestFeat]                            #最优特征的标签;第一个是房子
    featLabels.append(bestFeatLabel)
    myTree = {bestFeatLabel:{}}                                    #根据最优特征的标签生成树;{'有自己的房子':{}}
    del(labels[bestFeat])                                        #删除已经使用特征标签 ;把房子那一列特征删除
    featValues = [example[bestFeat] for example in dataSet]        #得到训练集中所有最优特征的属性值; 
    uniqueVals = set(featValues)                                #去掉重复的属性值
    for value in uniqueVals:                                    #遍历特征,创建决策树。                       
        myTree[bestFeatLabel][value] = createTree(splitDataSet(dataSet, bestFeat, value), labels, featLabels)  ⭐#假设第二列是最优特征,使用该特征作为根节点,进行递归,则原来的dataSet,会变成两个子dataSet,然后对这两个子dataSet分别进行递归创建树,直到满足结束条件。
    return myTree

if __name__ == '__main__':
    dataSet, labels = createDataSet()
    featLabels = []
    myTree = createTree(dataSet, labels, featLabels)
    print(myTree)


3 决策树可视化(回头补)


4 决策树分类

from math import log
import operator

def calcShannonEnt(dataSet):
    numEntires = len(dataSet)                     
    labelCounts = {}                            
    for featVec in dataSet:                          
        currentLabel = featVec[-1]                  
        if currentLabel not in labelCounts.keys():    
            labelCounts[currentLabel] = 0
        labelCounts[currentLabel] += 1              
    shannonEnt = 0.0                           
    for key in labelCounts:                        
        prob = float(labelCounts[key]) / numEntires 
        shannonEnt -= prob * log(prob, 2)            
    return shannonEnt                            

def createDataSet():
    dataSet = [[0, 0, 0, 0, 'no'],                     
            [0, 0, 0, 1, 'no'],
            [0, 1, 0, 1, 'yes'],
            [0, 1, 1, 0, 'yes'],
            [0, 0, 0, 0, 'no'],
            [1, 0, 0, 0, 'no'],
            [1, 0, 0, 1, 'no'],
            [1, 1, 1, 1, 'yes'],
            [1, 0, 1, 2, 'yes'],
            [1, 0, 1, 2, 'yes'],
            [2, 0, 1, 2, 'yes'],
            [2, 0, 1, 1, 'yes'],
            [2, 1, 0, 1, 'yes'],
            [2, 1, 0, 2, 'yes'],
            [2, 0, 0, 0, 'no']]
    labels = ['年龄', '有工作', '有自己的房子', '信贷情况']        
    return dataSet, labels                         

def splitDataSet(dataSet, axis, value):       
    retDataSet = []                                      
    for featVec in dataSet:                           
        if featVec[axis] == value:
            reducedFeatVec = featVec[:axis]             
            reducedFeatVec.extend(featVec[axis+1:])    
            retDataSet.append(reducedFeatVec)
    return retDataSet                                     

def chooseBestFeatureToSplit(dataSet):
    numFeatures = len(dataSet[0]) - 1                  
    baseEntropy = calcShannonEnt(dataSet)              
    bestInfoGain = 0.0                                
    bestFeature = -1                                 
    for i in range(numFeatures):                       
        #获取dataSet的第i个所有特征
        featList = [example[i] for example in dataSet]
        uniqueVals = set(featList)                        
        newEntropy = 0.0                             
        for value in uniqueVals:                        
            subDataSet = splitDataSet(dataSet, i, value)         
            prob = len(subDataSet) / float(len(dataSet))           
            newEntropy += prob * calcShannonEnt(subDataSet)     
        infoGain = baseEntropy - newEntropy                   
        # print("第%d个特征的增益为%.3f" % (i, infoGain))          
        if (infoGain > bestInfoGain):                          
            bestInfoGain = infoGain                           
            bestFeature = i                                   
    return bestFeature                                           

def majorityCnt(classList):
    classCount = {}
    for vote in classList:                                      
        if vote not in classCount.keys():classCount[vote] = 0   
        classCount[vote] += 1
    sortedClassCount = sorted(classCount.items(), key = operator.itemgetter(1), reverse = True)        
    return sortedClassCount[0][0]                       

def createTree(dataSet, labels, featLabels):
    classList = [example[-1] for example in dataSet]          
    if classList.count(classList[0]) == len(classList):            
        return classList[0]
    if len(dataSet[0]) == 1:                                  
        return majorityCnt(classList)
    bestFeat = chooseBestFeatureToSplit(dataSet)               
    bestFeatLabel = labels[bestFeat]                          
    featLabels.append(bestFeatLabel)
    myTree = {bestFeatLabel:{}}                               
    del(labels[bestFeat])                                       
    featValues = [example[bestFeat] for example in dataSet]      
    uniqueVals = set(featValues)                              
    for value in uniqueVals:                                                
        myTree[bestFeatLabel][value] = createTree(splitDataSet(dataSet, bestFeat, value), labels, featLabels)
    return myTree

"""
函数说明:使用决策树分类

Parameters:
    inputTree - 已经生成的决策树
    featLabels - 存储选择的最优特征标签
    testVec - 测试数据列表,顺序对应最优特征标签
Returns:
    classLabel - 分类结果
"""
def classify(inputTree, featLabels, testVec):                                    
    firstStr = next(iter(inputTree))                                             #获取决策树结点;'有自己的房子'
    secondDict = inputTree[firstStr]                                             #下一个字典;{0: {'有工作': {0: 'no', 1: 'yes'}}, 1: 'yes'}
    featIndex = featLabels.index(firstStr)                                       # featIndex = 0       
    for key in secondDict.keys():                                                #key = 0
        if testVec[featIndex] == key:
            if type(secondDict[key]).__name__ == 'dict':                         #如果是字典类型则要继续递归判断
                classLabel = classify(secondDict[key], featLabels, testVec)
            else: classLabel = secondDict[key]                                    #返回结果值
    return classLabel

if __name__ == '__main__':
    dataSet, labels = createDataSet()
    featLabels = []
    myTree = createTree(dataSet, labels, featLabels)
    testVec = [0,0]                                        #测试数据
    result = classify(myTree, featLabels, testVec)
    if result == 'yes':
        print('放贷')
    if result == 'no':
        print('不放贷')


阅读更多
文章标签: python
个人分类: 机器学习
上一篇《机器学习实战》个人学习记录笔记(四)———决策树
下一篇《机器学习实战》个人学习记录笔记(六)———Sklearn决策树预测隐形眼镜类型
想对作者说点什么? 我来说一句

没有更多推荐了,返回首页

关闭
关闭