机器学习实战之支持向量机SMO
SMO理论基础
参考:SequentialMinimalOptimization-AFastAlgorithmforTrainingSupportVectorMachines.pdf
下载地址:
SMO原文
SMO简单版伪代码
SMO伪代码 创建一个alpha向量并将其初始化为0向量 当迭代次数小于最大迭代次数时(外循环): 对数据集中每一个数据向量(内循环): 如果该向量可以被优化: 随机选择另一个数据向量 同时优化这两个向量 若果这两个向量都不能被优化,退出内循环 若果所有的向量都没有被优化,增加迭代次数,继续下一次循环
''' 机器学习实战-支持向量机学习 ''' from numpy import * from time import sleep ''' 准备数据 每行数据【特征1,特征2,类别标签】 ''' def loadDataSet(fileName): dataMat = []; labelMat = [] fr = open(fileName) for line in fr.readlines(): lineArr = line.strip().split('\t') dataMat.append([float(lineArr[0]), float(lineArr[1])]) labelMat.append(float(lineArr[2])) return dataMat, labelMat ''' 辅助函数:随机选择不同于i的下标 ''' def selectJrand(i, m): j = i while (j == i): j = int(random.uniform(0, m)) return j ''' 剪辑后aj的解 ''' def clipAlpha(aj, H, L): if aj > H: aj = H if L > aj: aj = L return aj ''' SMO伪代码 创建一个alpha向量并将其初始化为0向量 当迭代次数小于最大迭代次数时(外循环): 对数据集中每一个数据向量(内循环): 如果该向量可以被优化: 随机选择另一个数据向量 同时优化这两个向量 若果这两个向量都不能被优化,退出内循环 若果所有的向量都没有被优化,增加迭代次数,继续下一次循环 ''' # a = [1,2,3] # b = [[1,2,3],[1,2,3],[1,2,3]] # labelMat = mat(b) # alphas = mat(a).transpose() # fXi = multiply(alphas,labelMat).T # print(fXi) # [[1 2 3] # [2 4 6] # [3 6 9]] #输入数据集,类别标签,常数,容错率,最大迭代次数 def smoSimple(dataMatIn, classLabels, C, toler, maxIter): dataMatrix = mat(dataMatIn); labelMat = mat(classLabels).transpose() b = 0; m,n = shape(dataMatrix) alphas = mat(zeros((m,1))) iter = 0 while (iter < maxIter): alphaPairsChanged = 0 for i in range(m): #预测的类别 fXi = float(multiply(alphas,labelMat).T*(dataMatrix*dataMatrix[i,:].T)) + b #误差 Ei = fXi - float(labelMat[i])#if checks if an example violates KKT conditions #如果误差较大,且【0,C】(值得优化) if ((labelMat[i]*Ei < -toler) and (alphas[i] < C)) or ((labelMat[i]*Ei > toler) and (alphas[i] > 0)): j = selectJrand(i,m) fXj = float(multiply(alphas,labelMat).T*(dataMatrix*dataMatrix[j,:].T)) + b Ej = fXj - float(labelMat[j]) alphaIold = alphas[i].copy(); alphaJold = alphas[j].copy(); if (labelMat[i] != labelMat[j]): L = max(0, alphas[j] - alphas[i]) H = min(C, C + alphas[j] - alphas[i]) else: L = max(0, alphas[j] + alphas[i] - C) H = min(C, alphas[j] + alphas[i]) if L==H: print("L==H"); continue eta = 2.0 * dataMatrix[i,:]*dataMatrix[j,:].T - dataMatrix[i,:]*dataMatrix[i,:].T - dataMatrix[j,:]*dataMatrix[j,:].T if eta >= 0: print("eta>=0"); continue alphas[j] -= labelMat[j]*(Ei - Ej)/eta alphas[j] = clipAlpha(alphas[j],H,L) if (abs(alphas[j] - alphaJold) < 0.00001): print("j not moving enough"); continue alphas[i] += labelMat[j]*labelMat[i]*(alphaJold - alphas[j])#update i by the same amount as j #the update is in the oppostie direction b1 = b - Ei- labelMat[i]*(alphas[i]-alphaIold)*dataMatrix[i,:]*dataMatrix[i,:].T - labelMat[j]*(alphas[j]-alphaJold)*dataMatrix[i,:]*dataMatrix[j,:].T b2 = b - Ej- labelMat[i]*(alphas[i]-alphaIold)*dataMatrix[i,:]*dataMatrix[j,:].T - labelMat[j]*(alphas[j]-alphaJold)*dataMatrix[j,:]*dataMatrix[j,:].T if (0 < alphas[i]) and (C > alphas[i]): b = b1 elif (0 < alphas[j]) and (C > alphas[j]): b = b2 else: b = (b1 + b2)/2.0 alphaPairsChanged += 1 print ("iter: %d i:%d, pairs changed %d" % (iter,i,alphaPairsChanged)) if (alphaPairsChanged == 0): iter += 1 else: iter = 0 #alphaPairsChanged = 1 说明还能优化,要迭代到不能优化 print( "iteration number: %d" % iter) return b,alphas
测试
import os from ML_Learn.com.ML.Class.SVM import svmMLiA #导入训练数据集 dataArr, labelArr = svmMLiA.loadDataSet(os.getcwd() + '/resource/testSet.txt') #简单版SMO测试 b,alphas = svmMLiA.smoSimple(dataArr,labelArr,0.6,0.001,40) print("b: " , b) print("alphas>0: ", alphas[alphas>0]) for i in range(100): if alphas[i]>0.0: print("支持向量元素:\t" ,dataArr[i] ,"\t" ,labelArr[i] ) # b: [[-3.84148046]] # alphas>0: [[ 0.14709994 0.17249089 0.04916758 0.00392681 0.36483161]] # 支持向量元素: [4.658191, 3.507396] -1.0 # 支持向量元素: [3.457096, -0.082216] -1.0 # 支持向量元素: [2.893743, -1.643468] -1.0 # 支持向量元素: [5.286862, -2.358286] 1.0 # 支持向量元素: [6.080573, 0.418886] 1.0