1、算法思想:
下图中,方形框为数据集的特征,0/1为特征下的值,圆形框中为数据的最终类别。目的就是给数据集构造如上图的决策树,再用其为其它数据进行分类。
H(C):数据集C的香农熵,H(C|T):数据集C用特征T来划分后的香农熵,IG(T):为由原来数据C到划分集的信息增益
从树根越往叶子节点走,数据集的类别就会越确定(从混乱到有序的过程),那这个数据集的香农熵就会越来越小,所以在决定一个数据集的最好划分特征时,可以取划分后信息增益最大的特征作为最好划分特征。
2、程序实现:
from math import log
import operator
def createDataSet():
dataSet = [[1, 1, 'yes'],
[1, 1, 'yes'],
[1, 0, 'no'],
[0, 1, 'no'],
[0, 1, 'no']]
labels = ['no surfacing','flippers']
return dataSet, labels
def calcShannonEnt(dataSet):#计算数据集的香农熵
numEntries = len(dataSet)
labelCounts = {}
for featVec in dataSet:
currentLabel = featVec[-1]
if currentLabel not in labelCounts.keys(): labelCounts[currentLabel] = 0
labelCounts[currentLabel] += 1
shannonEnt = 0.0
for key in labelCounts:
prob = float(labelCounts[key])/numEntries
shannonEnt -= prob * log(prob,2) #log base 2
return shannonEnt
def splitDataSet(dataSet, axis, value): #用指定特征的某个值划分数据
retDataSet = []
for featVec in dataSet:
if featVec[axis] == value:
reducedFeatVec = featVec[:axis]
reducedFeatVec.extend(featVec[axis+1:])
retDataSet.append(reducedFeatVec)
return retDataSet
def chooseBestFeatureToSplit(dataSet): #求数据集的最好的划分特征
numFeatures = len(dataSet[0]) - 1
baseEntropy = calcShannonEnt(dataSet)
bestInfoGain = 0.0; bestFeature = -1
for i in range(numFeatures):
featList = [example[i] for example in dataSet]
uniqueVals = set(featList)
newEntropy = 0.0
for value in uniqueVals:
subDataSet = splitDataSet(dataSet, i, value)
prob = len(subDataSet)/float(len(dataSet))
newEntropy += prob * calcShannonEnt(subDataSet)
infoGain = baseEntropy - newEntropy
if (infoGain > bestInfoGain):
bestInfoGain = infoGain
bestFeature = i
return bestFeature
def createTree(dataSet,labels): #创建树
classList = [example[-1] for example in dataSet]
#两个停止条件
if classList.count(classList[0]) == len(classList):
return classList[0]
if len(dataSet[0]) == 1:
return majorityCnt(classList)
bestFeat = chooseBestFeatureToSplit(dataSet) #找到这次数据的划分特征
bestFeatLabel = labels[bestFeat]
myTree = {bestFeatLabel:{}} #创建嵌套字典
del(labels[bestFeat]) #更新特征集
featValues = [example[bestFeat] for example in dataSet]
uniqueVals = set(featValues) #统计特征值
for value in uniqueVals: #按照这些值分别再对键值求值(满足停条后返回的值)和递归求支树
subLabels = labels[:]
myTree[bestFeatLabel][value] = createTree(splitDataSet(dataSet, bestFeat, value),subLabels)
return myTree
def majorityCnt(classList): # 投票表决,classList为分类名称的列表
classCount = {}
for vote in classList: # 创建字典存储每个类出现次数
if vote not in classCount.keys(): classCount[vote] = 0
classCount[vote] += 1
sortedClassCount = sorted(classCount.iteritems(), key=operator.itemgetter(1), reverse=True) #排序
return sortedClassCount[0][0] #取最高频率
def classify(inputTree,featLabels,testVec): #使用决策树inputTree对testVec进行分类,featLabels为testVec的特征集
#firstStr = inputTree.keys()[0]
firstSides = list(inputTree.keys()) #先转换成list,再把需要的索引提取出来
firstStr = firstSides[0]
secondDict = inputTree[firstStr] #键(根)的值(所有支树)
featIndex = featLabels.index(firstStr) #找到根特征在testVec的位置
key = testVec[featIndex] #testVec在根特征下的值
valueOfFeat = secondDict[key] #树的特征值(也是键)的值
if isinstance(valueOfFeat, dict): #如果存在支树
classLabel = classify(valueOfFeat, featLabels, testVec)
else: classLabel = valueOfFeat
return classLabel
def storeTree(inputTree,filename): #存储树到文件
import pickle
fw = open(filename,'w')
pickle.dump(inputTree,fw)
fw.close()
def grabTree(filename): #从文件中读取树
import pickle
fr = open(filename)
return pickle.load(fr)
测试1:
mydata,myleb=createDataSet()
mytree=createTree(mydata,myleb)
print(type(mytree))
storeTree(mytree,'ceshi.tst')#存储树到文件
mytree1=grabTree('ceshi.tst')#再从文件中读取树
print("训练出来的树是:")
print(mytree1)
mytest=[0,0] #用于测试
mydata1,myleb1=createDataSet() #在createTree中myleb1的值会被改
myend=classify(mytree,myleb1,mytest)
print("测试案例:")
print(mytest)
print("的结果为:"+myend)
运行结果:
测试2(从文件中取出训练数据,构造并使用决策树预测隐形眼镜类型):
fr=open('lenses.txt')
lenses=[inst.strip().split('\t') for inst in fr.readlines()] #训练数据
lenseslabels=['age','prescript','astigmatic','tearRate'] #数据特征
lensestree=createTree(lenses,lenseslabels) #构造树
storeTree(lensestree, 'wenjian') # 存储树到文件
mytree = grabTree('wenjian') # 再从文件中读取树
print("训练出来的树是:")
print(mytree)
#####################开始测试
mytest = ['presbyopic','hyper', 'no', 'normal'] # 用于测试
print("测试案例:")
print(mytest)
lenseslabels = ['age', 'prescript', 'astigmatic', 'tearRate'] # 在createTree中leb1的值会被改
myend = classify(mytree, lenseslabels, mytest) # 使用决策树对mytest进行分类
print("结果为:" + myend)
运行结果:
3、决策树的过拟合
解决方法:剪枝
由全树的T0开始,剪枝部分结点得到T1,再减可得T2,T3......Tk, 在验证数据时对这K个树分别测试得出结果,选择损失函数最小的数T
4、补充:
1、随机森林
2、其他构造决策树方法:基尼系数、增益率