集成方法的关键是开发出一种可以生成大量近似独立的模型的方法，然后把它们集成起来。集成方法是由两层算法组成的层次架构。底层的算法叫作基学习器。目前广泛使用的上层算法主要有：投票（bagging）、提升（boosting）和随机森林（random forests）。严格地讲，随机森林实际上是上层算法和修改后的二元决策树的组合。
有很多算法都可以用作基学习器，如二元决策树、支持向量机等，但从实用角度二元决策树的应用最为广泛。
6.1 二元决策树
__author__ = 'mike-bowles'
import urllib.request as urllib2
import numpy
from sklearn import tree
from sklearn.tree import DecisionTreeRegressor
from sklearn.externals.six import StringIO
from math import sqrt
import matplotlib.pyplot as plot
from io import StringIO as pyIO
import csv
target_url = ("http://archive.ics.uci.edu/ml/machine-learning-databases/wine-quality/winequality-red.csv")
data = urllib2.urlopen(target_url)
data = data.read().decode('utf-8')
dataFile = pyIO(data)
csvReader = csv.reader(dataFile)
xList = []
labels = []
names = []
firstLine = True
for line in csvReader:
#print(line)
line = line[0]
if firstLine:
names = line.strip().split(";")
firstLine = False
else:
# split on semi-colon
row = line.strip().split(";")
labels.append(float(row[-1]))
row.pop()
floatRow = [float(num) for num in row]
xList.append(floatRow)
nrows = len(xList)
ncols = len(xList[0])
wineTree = DecisionTreeRegressor(max_depth=3)
wineTree.fit(xList, labels)
with open("wineTree.dot", 'w') as f:
f = tree.export_graphviz(wineTree, out_file=f)
#dot -Tpng wineTree.dot -o wineTree.png
如何训练一个二元决策树：
__author__ = "mike-bowles"
import numpy
import matplotlib.pyplot as plot
from sklearn import tree
from sklearn.tree import DecisionTreeRegressor
from sklearn.externals.six import StringIO
nPoints = 100
xPlot = [(float(i)/float(nPoints)-0.5) for i in range(nPoints+1)]
x = [[s] for s in xPlot]
numpy.random.seed(1)
y = [s + numpy.random.normal(scale=0.1) for s in xPlot]
plot.plot(xPlot, y)
plot.axis('tight')
plot.xlabel('x')
plot.ylabel('y')
plot.show()
simpleTree = DecisionTreeRegressor(max_depth=1)
simpleTree.fit(x, y)
with open("simpleTree.dot", 'w') as f:
f = tree.export_graphviz(simpleTree, out_file=f)
#
yHat = simpleTree.predict(x)
print(yHat)
plot.figure()
plot.plot(xPlot, y, label="True y")
plot.plot(xPlot, yHat, label="Tree Prediction ", linestyle="--")
plot.legend(bbox_to_anchor=(1,0.2))
plot.axis('tight')
plot.xlabel('x')
plot.ylabel('y')
plot.show()
simpleTree2 = DecisionTreeRegressor(max_depth=2)
simpleTree2.fit(x, y)
with open("simpleTree2.dot", 'w') as f:
f = tree.export_graphviz(simpleTree2, out_file=f)
yHat = simpleTree2.predict(x)
print(yHat)
plot.figure()
plot.plot(xPlot, y, label="True y")
plot.plot(xPlot, yHat, label="Tree Prediction ", linestyle="--")
plot.legend(bbox_to_anchor=(1, 0.2))
plot.axis("tight")
plot.xlabel('x')
plot.ylabel('y')
plot.show()
# find the best one
sse = []
xMin = []
for i in range(1, len(xPlot)):
# divide list into points
lhList = list(xPlot[0:i])
rhList = list(xPlot[i:len(xPlot)])
# calculate
lhAvg = sum(lhList) / len(lhList)
rhAvg = sum(rhList) / len(rhList)
# cal
lhSse = sum([(s-lhAvg)*(s-lhAvg) for s in lhList])
rhSse = sum([(s-rhAvg)*(s-rhAvg) for s in rhList])
# add sum
sse.append(lhSse + rhSse)
xMin.append(max(lhList))
plot.plot(range(1, len(xPlot)), sse)
plot.xlabel('Split Point Index')
plot.ylabel('Sum Squared Error')
plot.show()
minSse = min(sse)
idxMin = sse.index(minSse)
print(xMin[idxMin])
# what happens if the depth is really high?
simpleTree6 = DecisionTreeRegressor(max_depth=6)
simpleTree6.fit(x, y)
yHat = simpleTree6.predict(x)
print(yHat)
plot.figure()
plot.plot(xPlot, y, label='True y')
plot.plot(xPlot, yHat, label="Tree Prediction ", linestyle="-")
plot.legend(bbox_to_anchor=(1,0.2))
plot.axis('tight')
plot.xlabel('x')
plot.ylabel('y')
plot.show()
上图为depth=6的关系图，正如预期，y值大致上一直随x值变化，但是有些随机的小扰动
决策树的训练等同于分割点的选择：训练一个决策树需要穷尽地搜索所有可能的分割点来确定哪个值可以使误差平方和最小化。
多变量决策树的训练-选择哪个属性进行分割：算法会对所有的属性检查所有可能的分割点，对每个属性找打误差平方和最小的分割点，然后找到哪个属性对应的误差平方和最小。
决策树训练算法通常有一个参数来控制节点包含的数据实例最小到什么规模就不再分割。节点包含的数据实例太少会导致预测结果发生剧烈震荡。
二元决策树的过拟合：在实际问题中，使用交叉验证（cross-validation）来控制过拟合，
__author__ = 'mike-bowles'
import numpy
import matplotlib.pyplot as plot
from sklearn import tree
from sklearn.tree import DecisionTreeRegressor
from sklearn.externals.six import StringIO
nPoints = 100
xPlot = [(float(i)/float(nPoints) - 0.5) for i in range(nPoints + 1)]
x = [[s] for s in xPlot]
numpy.random.seed(1)
y = [s + numpy.random.normal(scale=0.1) for s in xPlot]
nrow = len(x)
# different depth
depthList = [1, 2, 3, 4, 5, 6, 7]
xvalMSE = []
nxval = 10
for iDepth in depthList:
# build cross-validation
for ixval in range(nxval):
idxTest = [a for a in range(nrow) if a%nxval==ixval%nxval]
idxTrain = [a for a in range(nrow) if a%nxval!=ixval%nxval]
#
xTrain = [x[r] for r in idxTrain]
xTest = [x[r] for r in idxTest]
yTrain = [y[r] for r in idxTrain]
yTest = [y[r] for r in idxTest]
# train tree
treeModel = DecisionTreeRegressor(max_depth=iDepth)
treeModel.fit(xTrain, yTrain)
#
treePrediction = treeModel.predict(xTest)
error = [yTest[r] - treePrediction[r] for r in range(len(yTest))]
# accumulate squared errors
if ixval == 0:
oosErrors = sum([e * e for e in error])
else:
oosErrors += sum([e * e for e in error])
mse = oosErrors/nrow
xvalMSE.append(mse)
plot.plot(depthList, xvalMSE)
plot.axis('tight')
plot.xlabel('Tree Depth')
plot.ylabel('Mean Squared Error')
plot.show()
数据点增加到1000时
单个决策树时出现的问题（如需要调整多个参数、结果的不稳定性、决策树深度加深导致的过拟合等）就会减弱。这也是提出集成方法的原因，集成方法更加鲁棒、易于训练、更加准确。下面讨论三个主流的集成方法。
6.2 自举集成：Bagging算法
该方法是使用叫作自举（bootstrap）的取样方法，通常从一个中等规模的数据集中产生取样统计。自举集成从训练数据集中获得一系列的自举样本，然后针对每一个自举样本训练一个基学习器。对于回归问题，结果为基学习器的均值。对于分类问题，结果是从不同类别所占的百分比引申出来的各种类别的概率或均值。
代码预留30%的数据作为测试数据，以代替交叉验证方法。参数numTreesMax决定集成方法包含的决策树的最大数目。代码建立模型是从第一决策树开始，然后是前两个决策树，前三个决策树，以此类推，直到numTreesMax个决策树，可以看到预测的准确性与决策树数目之间的关系。代码将训练好的模型存入一个列表，并且存储了测试数据的预测值，这些预测值用于评估测试误差。
__author__ = 'mike-bowles'
import numpy
import matplotlib.pyplot as plot
from sklearn import tree
from sklearn.tree import DecisionTreeRegressor
from math import floor
import random
nPoints = 1000
xPlot = [(float(i)/float(nPoints)-0.5) for i in range(nPoints+1)]
x = [[s] for s in xPlot]
random.seed(1)
y = [s + numpy.random.normal(scale=0.1) for s in xPlot]
# take fixed test set 30% of sample
nSample = int(nPoints * 0.30)
idxTest = random.sample(range(nPoints), nSample)
idxTest.sort()
idxTrain = [idx for idx in range(nPoints) if not(idx in idxTest)]
xTrain = [x[r] for r in idxTrain]
xTest = [x[r] for r in idxTest]
yTrain = [y[r] for r in idxTrain]
yTest = [y[r] for r in idxTest]
numTreesMax = 20
treeDepth = 1
modelList = []
predList = []
# number of samples to draw for stochastic bagging
nBagSamples = int(len(xTrain) * 0.5)
for iTrees in range(numTreesMax):
idxBag = random.sample(range(len(xTrain)), nBagSamples)
xTrainBag = [xTrain[i] for i in idxBag]
yTrainBag = [yTrain[i] for i in idxBag]
modelList.append(DecisionTreeRegressor(max_depth=treeDepth))
modelList[-1].fit(xTrainBag, yTrainBag)
# make prediction
latestPrediction = modelList[-1].predict(xTest)
predList.append(list(latestPrediction))
# build cumulative
mse = []
allPredictions = []
for iModels in range(len(modelList)):
prediction = []
for iPred in range(len(xTest)):
prediction.append(sum([predList[i][iPred] for i in range(iModels+1)])/(iModels+1))
allPredictions.append(prediction)
errors = [(yTest[i] - prediction[i]) for i in range(len(yTest))]
mse.append(sum([e*e for e in errors])/len(yTest))
nModels = [i+1 for i in range(len(modelList))]
plot.plot(nModels, mse)
plot.axis('tight')
plot.xlabel('Number of Models in Ensemble')
plot.ylabel('Mean Squared Error')
plot.ylim((0.0, max(mse)))
plot.show()
plotList = [0, 9, 19]
for iPlot in plotList:
plot.plot(xTest, allPredictions[iPlot])
plot.plot(xTest, yTest, linestyle="--")
plot.axis('tight')
plot.xlabel('x value')
plot.ylabel('Predictions')
plot.show()
Bagging 的性能-偏差与方差：当更多的数据点加入时，并不能减少的误差叫作偏差。深度为1的决策树的偏差来自于模型太简单了，Bagging方法减少了模型之间的方差。但是对深度为1的决策树的偏差错误则是平均不掉的。克服这个为题的方法就是增加决策树的深度。
当采用深度为5的决策树时，均方误差稍微小于0.01（可能是由于噪声数据的随机性）
从上图可以看出，单个决策树的预测结果明显比其他的突出，因为它有几个尖锐的突出，这导致了严重的误差。换句话说，单个决策树有高的方差。其他单个的决策树毫无疑问具有相似的性能。但是当它们被平均时（采用集成方法），方差减少了；基于Bagging算法的预测曲线更光滑，更接近于真实值。
Bagging算法如何解决多变量问题：
__author__ = 'mike-bowles'
import urllib.request as urllib2
import numpy
from sklearn import tree
from sklearn.tree import DecisionTreeRegressor
import random
from math import sqrt
import matplotlib.pyplot as plot
import csv
from io import StringIO as pyIO
target_url = ("http://archive.ics.uci.edu/ml/machine-learning-databases/wine-quality/winequality-red.csv")
data = urllib2.urlopen(target_url)
data = data.read().decode('utf-8')
dataFile = pyIO(data)
csvReader = csv.reader(dataFile)
xList = []
labels = []
names = []
firstLine = True
for line in csvReader:
if firstLine:
print(line)
firstLine = False
continue
row = line[0].strip().split(";")
labels.append(float(row[-1]))
row.pop()
floatRow = [float(num) for num in row]
xList.append(floatRow)
nrows = len(xList)
ncols = len(xList[0])
random.seed(1)
nSample = int(nrows * 0.30)
idxTest = random.sample(range(nrows), nSample)
idxTest.sort()
idxTrain = [idx for idx in range(nrows) if not(idx in idxTest)]
# Define
xTrain = [xList[r] for r in idxTrain]
xTest = [xList[r] for r in idxTest]
yTrain = [labels[r] for r in idxTrain]
yTest = [labels[r] for r in idxTest]
# max
numTreesMax = 30
treeDepth = 12
modelList = []
predList = []
nBagSamples = int(len(xTrain) * 0.5)
for iTrees in range(numTreesMax):
idxBag = []
for i in range(nBagSamples):
idxBag.append(random.choice(range(len(xTrain))))
xTrainBag = [xTrain[i] for i in idxBag]
yTrainBag = [yTrain[i] for i in idxBag]
modelList.append(DecisionTreeRegressor(max_depth=treeDepth))
modelList[-1].fit(xTrainBag, yTrainBag)
# make
latestPrediction = modelList[-1].predict(xTest)
predList.append(list(latestPrediction))
mse = []
allPredictions = []
for iModels in range(len(modelList)):
# average first
prediction = []
for iPred in range(len(xTest)):
prediction.append(sum([predList[i][iPred] for i in range(iModels+1)]) / (iModels+1))
allPredictions.append(prediction)
errors = [(yTest[i] - prediction[i]) for i in range(len(yTest))]
mse.append(sum([e*e for e in errors])/len(yTest))
nModels = [i + 1 for i in range(len(modelList))]
plot.plot(nModels, mse)
plot.axis('tight')
plot.xlabel("Number of Tree Models in Ensembel")
plot.ylabel("Mean Squared Error")
plot.ylim((0.0, max(mse)))
plot.show()
print("Minimum MSE")
print(min(mse))以下是深度为12
Bagging方法可以作为集成方法的入门介绍，因为它比较简单，易于理解，而且易于证明他可以减少方差的特性。Bagging不常用
6.3 梯度提升法（Gradient Boosting）
梯度提升法是基于决策树的集成方法，在不同标签上训练决策树，然后将其组合起来。对于回归问题，目标是最小化均方误差，每个后续的决策树是在前面决策树遗留的错误上进行训练。
基本原理：梯度提升法与Bagging和随机森林的不同之处在于它在减少方差的同时，还可以减少偏差。梯度提升法的特性就是它在树桩（深度为1）的决策树的情况下，也可以获得同更深的决策树一样低的均方误差值。对于梯度提升法，只有属性之间强烈的相互影响的情况下，才需要考虑增加决策树的深度。随着决策树深度的增加，性能获得了改善实际上可以作为判断属性之间是否存在相互影响的方法。
__author__ = 'mike-bowles'
import numpy
import matplotlib.pyplot as plot
from sklearn import tree
from sklearn.tree import DecisionTreeRegressor
from math import floor
import random
nPoints = 1000
xPlot = [(float(i)/float(nPoints)-0.5) for i in range(nPoints+1)]
x = [[s] for s in xPlot]
random.seed(1)
y = [s + numpy.random.normal(scale=0.1) for s in xPlot]
# take fixed test set 30% of sample
nSample = int(nPoints * 0.30)
idxTest = random.sample(range(nPoints), nSample)
idxTest.sort()
idxTrain = [idx for idx in range(nPoints) if not(idx in idxTest)]
xTrain = [x[r] for r in idxTrain]
xTest = [x[r] for r in idxTest]
yTrain = [y[r] for r in idxTrain]
yTest = [y[r] for r in idxTest]
numTreesMax = 30
treeDepth = 1
modelList = []
predList = []
eps = 0.3
residuals = list(yTrain)
for iTrees in range(numTreesMax):
modelList.append(DecisionTreeRegressor(max_depth=treeDepth))
modelList[-1].fit(xTrain, residuals)
latestInSamplePrediction = modelList[-1].predict(xTrain)
residuals = [residuals[i] - eps*latestInSamplePrediction[i] for i in range(len(residuals))] # !!!!
latestInSamplePrediction = modelList[-1].predict(xTest)
predList.append(list(latestInSamplePrediction))
mse = []
allPredictions = []
for iModels in range(len(modelList)):
prediction = []
for iPred in range(len(xTest)):
prediction.append(sum([predList[i][iPred] for i in range(iModels+1)]) * eps)
allPredictions.append(prediction)
errors = [(yTest[i] - prediction[i]) for i in range(len(yTest))]
mse.append(sum([e*e for e in errors]) / len(yTest))
nModels = [i+1 for i in range(len(modelList))]
plot.plot(nModels, mse)
plot.axis('tight')
plot.xlabel('Number of Models in Ensemble')
plot.ylabel('Mean Squared Error')
plot.ylim((0.0, max(mse)))
plot.show()
plotList = [0, 14, 29]
lineType = [':', '-.', '--']
plot.figure()
for i in range(len(plotList)):
iPlot = plotList[i]
textLegend = 'Prediction with ' + str(iPlot) + ' Trees'
plot.plot(xTest, allPredictions[iPlot], label = textLegend, linestyle = lineType[i])
plot.plot(xTest, yTest, label="True y Value", alpha=0.25)
plot.legend(bbox_to_anchor=(1,0.3))
plot.axis('tight')
plot.xlabel('x value')
plot.ylabel('Predictions')
plot.show()
针对多变量问题的梯度提升法：可通过调整参数改善性能
__author__ = 'mike-bowles'
import urllib.request as urllib2
import numpy
from sklearn import tree
from sklearn.tree import DecisionTreeRegressor
import random
from math import sqrt
import matplotlib.pyplot as plot
import csv
from io import StringIO as pyIO
target_url = ("http://archive.ics.uci.edu/ml/machine-learning-databases/wine-quality/winequality-red.csv")
data = urllib2.urlopen(target_url)
data = data.read().decode('utf-8')
dataFile = pyIO(data)
csvReader = csv.reader(dataFile)
xList = []
labels = []
names = []
firstLine = True
for line in csvReader:
if firstLine:
print(line)
firstLine = False
continue
row = line[0].strip().split(";")
labels.append(float(row[-1]))
row.pop()
floatRow = [float(num) for num in row]
xList.append(floatRow)
nrows = len(xList)
ncols = len(xList[0])
random.seed(1)
nSample = int(nrows * 0.30)
idxTest = random.sample(range(nrows), nSample)
idxTest.sort()
idxTrain = [idx for idx in range(nrows) if not(idx in idxTest)]
# Define
xTrain = [xList[r] for r in idxTrain]
xTest = [xList[r] for r in idxTest]
yTrain = [labels[r] for r in idxTrain]
yTest = [labels[r] for r in idxTest]
# max
numTreesMax = 30
treeDepth = 5
modelList = []
predList = []
eps = 0.1
residuals = list(yTrain)
for iTrees in range(numTreesMax):
modelList.append(DecisionTreeRegressor(max_depth=treeDepth))
modelList[-1].fit(xTrain, residuals)
latestInSamplePrediction = modelList[-1].predict(xTrain)
residuals = [residuals[i] - eps*latestInSamplePrediction[i] for i in range(len(residuals))] # !!!!
latestInSamplePrediction = modelList[-1].predict(xTest)
predList.append(list(latestInSamplePrediction))
mse = []
allPredictions = []
for iModels in range(len(modelList)):
prediction = []
for iPred in range(len(xTest)):
prediction.append(sum([predList[i][iPred] for i in range(iModels+1)]) * eps)
allPredictions.append(prediction)
errors = [(yTest[i] - prediction[i]) for i in range(len(yTest))]
mse.append(sum([e*e for e in errors]) / len(yTest))
nModels = [i+1 for i in range(len(modelList))]
plot.plot(nModels, mse)
plot.axis('tight')
plot.xlabel('Number of Models in Ensemble')
plot.ylabel('Mean Squared Error')
plot.ylim((0.0, max(mse)))
plot.show()
print('Minimum MSE')
print(min(mse))
Bagging和梯度提升法在工作原理上的根本差异在于梯度提升法持续监测自己的累积误差，然后使用残差进行后续训练。这种根本差异也解释了为什么当问题属性之间存在强的相互依赖、相互作用时，梯度提升法只需要调整决策树的深度。
6.4 随机森林
随机森林在数据集的子集上训练出一系列的模型。这些子集是从全训练数据集中随机抽取的。一种抽取方法就是对数据行的随机放回取样，同Bagging方法一样。另一种方法是每个决策树的训练数据集只是所有属性随机抽取的一个子集，而不是全部的属性。
算法的提出者建议对于回归问题，选择全部属性的三分之一，对于分类问题，选择全部属性数目的平方根。
__author__ = 'mike-bowles'
import urllib.request as urllib2
import numpy
from sklearn import tree
from sklearn.tree import DecisionTreeRegressor
import random
from math import sqrt
import matplotlib.pyplot as plot
import csv
from io import StringIO as pyIO
target_url = ("http://archive.ics.uci.edu/ml/machine-learning-databases/wine-quality/winequality-red.csv")
data = urllib2.urlopen(target_url)
data = data.read().decode('utf-8')
dataFile = pyIO(data)
csvReader = csv.reader(dataFile)
xList = []
labels = []
names = []
firstLine = True
for line in csvReader:
if firstLine:
print(line)
firstLine = False
continue
row = line[0].strip().split(";")
labels.append(float(row[-1]))
row.pop()
floatRow = [float(num) for num in row]
xList.append(floatRow)
nrows = len(xList)
ncols = len(xList[0])
random.seed(1)
nSample = int(nrows * 0.30)
idxTest = random.sample(range(nrows), nSample)
idxTest.sort()
idxTrain = [idx for idx in range(nrows) if not(idx in idxTest)]
# Define
xTrain = [xList[r] for r in idxTrain]
xTest = [xList[r] for r in idxTest]
yTrain = [labels[r] for r in idxTrain]
yTest = [labels[r] for r in idxTest]
# max
numTreesMax = 30
treeDepth = 1
modelList = []
predList = []
nAttr = 4
indexList = []
nTrainRows = len(yTrain)
for iTrees in range(numTreesMax):
modelList.append(DecisionTreeRegressor(max_depth=treeDepth))
idxAttr = random.sample(range(ncols), nAttr)
idxAttr.sort()
indexList.append(idxAttr)
#
idxRows = []
for i in range(int(0.5*nTrainRows)):
idxRows.append(random.choice(range(len(xTrain))))
idxRows.sort()
xRfTrain = []
yRfTrain = []
for i in range(len(idxRows)):
temp = [xTrain[idxRows[i]][j] for j in idxAttr] # !!!!
xRfTrain.append(temp)
yRfTrain.append(yTrain[idxRows[i]])
modelList[-1].fit(xRfTrain, yRfTrain)
xRfTest = []
for xx in xTest:
temp = [xx[i] for i in idxAttr]
xRfTest.append(temp)
latestOutSamplePrediction = modelList[-1].predict(xRfTest)
predList.append(list(latestOutSamplePrediction))
mse = []
allPredictions = []
for iModels in range(len(modelList)):
prediction = []
for iPred in range(len(xTest)):
prediction.append(sum([predList[i][iPred] for i in range(iModels+1)]))
allPredictions.append(prediction)
errors = [(yTest[i] - prediction[i]) for i in range(len(yTest))]
mse.append(sum([e*e for e in errors]) / len(yTest))
nModels = [i+1 for i in range(len(modelList))]
plot.plot(nModels, mse)
plot.axis('tight')
plot.xlabel('Number of Models in Ensemble')
plot.ylabel('Mean Squared Error')
plot.ylim((0.0, max(mse)))
plot.show()
print('Minimum MSE')
print(min(mse))随机森林是两个方法的结合，包括Bagging方法和属性随机选择方法。属性随机选择方法实际上是对二元决策树基学习器的修正。有研究结果建议随机森林更适用于广泛稀疏的属性空间，如文本挖掘问题。
1175
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
