一.最优化与计算复杂性:以最小的代价取得最大的收益(矩阵分析,数值分析,计算机(GPU并行计算，云计算，云存储))----数学建模、公式推导、算法设计。机器学习中的多数算法都是针对NP类问题(包括NP完全性问题):背包问题，最短路径问题，TSP问题，最大团问题，图同构问题等。
二.梯度下降法
梯度法是求解无约束多源函数值的最早的数值方法，很多机器学习的常用算法都是以它作为算法框架，进行改进和修正而导出更为复杂的优化算法。Logistic函数---世界不是非黑即白。
函数表达式:
(1)运用其作为线性分类器:
# -*- coding: utf-8 -*-
# Filename : Recommand_lib.py
from numpy import *
import numpy as np
import operator
# import scipy.spatial.distance.cdist as dist
import matplotlib.pyplot as plt
def savefile(savepath,content):
fp = open(savepath,"wb")
fp.write(content)
fp.close()
# 数据文件转矩阵
# path: 数据文件路径
# delimiter: 文件分隔符
def file2matrix(path,delimiter):
recordlist = []
fp = open(path,"rb") # 读取文件内容
content = fp.read()
fp.close()
rowlist = content.splitlines() # 按行转换为一维表
# 逐行遍历 # 结果按分隔符分割为行向量
recordlist=[map(eval, row.split(delimiter)) for row in rowlist if row.strip()]
return mat(recordlist) # 返回转换后的矩阵形式
# 欧氏距离
eps = 1.0e-6
def distEclud(vecA, vecB):
return linalg.norm(vecA-vecB)+eps
# 相关系数
def distCorrcoef(vecA, vecB):
return corrcoef(vecA, vecB, rowvar = 0)[0][1]
# Jaccard距离
# def distJaccard(vecA, vecB):
# temp = mat([array(vecA.tolist()[0]),array(vecB.tolist()[0])])
# return dist.pdist(temp,'jaccard')
# 余弦相似度
def cosSim(vecA, vecB):
return (dot(vecA,vecB.T)/((linalg.norm(vecA)*linalg.norm(vecB))+eps))[0,0]
# 绘制散点图
def drawScatter(plt,mydata,size=20,color='blue',mrkr='o'):
m,n = shape(mydata)
if m>n and m>2:
plt.scatter(mydata.T[0],mydata.T[1],s=size,c=color,marker=mrkr)
else:
plt.scatter(mydata[0],mydata[1],s=size,c=color,marker=mrkr)
# 绘制分类点
def drawScatterbyLabel(plt,Input):
m,n=shape(Input)
target = Input[:,-1]
for i in xrange(m):
if target[i]==0:
plt.scatter(Input[i,0],Input[i,1],c='blue',marker='o')
else:
plt.scatter(Input[i,0],Input[i,1],c='red',marker='s')
# 硬限幅函数
def hardlim(dataSet):
dataSet[nonzero(dataSet.A>0)[0]]=1
dataSet[nonzero(dataSet.A<=0)[0]]=0
return dataSet
# Logistic函数
def logistic(wTx):
return 1.0/(1.0+exp(-wTx))
def buildMat(dataSet):
m,n=shape(dataSet)
dataMat = zeros((m,n))
dataMat[:,0] = 1
dataMat[:,1:] = dataSet[:,:-1]
return dataMat
# 分类函数
def classifier(testData, weights):
prob = logistic(sum(testData*weights)) # 求取概率--判别算法
if prob > 0.5:
return 1.0 # prob>0.5 返回为1
else:
return 0.0 # prob<=0.5 返回为0
# 最小二乘回归，用于测试
def standRegres(xArr,yArr):
xMat = mat(ones((len(xArr),2)))
yMat = mat(ones((len(yArr),1)))
xMat[:,1:] = (mat(xArr).T)[:,0:]
yMat[:,0:] = (mat(yArr).T)[:,0:]
xTx = xMat.T*xMat
if linalg.det(xTx) == 0.0:
print "This matrix is singular, cannot do inverse"
return
ws = xTx.I * (xMat.T*yMat)
return ws# -*- coding: utf-8 -*-
import os
import sys
import numpy as np
import operator
from numpy import *
from common_libs import *
import matplotlib.pyplot as plt
# 1.导入数据
Input = file2matrix("testSet.txt","\t")
target = Input[:,-1] #获取分类标签列表
[m,n] = shape(Input)
# 2.按分类绘制散点图
drawScatterbyLabel(plt,Input)
# 3.构建b+x 系数矩阵：b这里默认为1
dataMat = buildMat(Input)
print dataMat[:10,:]
# 4. 定义步长和迭代次数
alpha = 0.001 # 步长
steps = 500 # 迭代次数
weights = ones((n,1))# 初始化权重向量
# 5. 主程序
for k in xrange(steps):
gradient = dataMat*mat(weights) # 梯度
output = logistic(gradient) # logistic函数
errors = target-output # 计算误差
weights = weights + alpha*dataMat.T*errors
print weights # 输出训练后的权重
# 6. 绘制训练后超平面
X = np.linspace(-7,7,100)
#y=w*x+b: b:weights[0]/weights[2]; w:weights[1]/weights[2]
Y = -(double(weights[0])+X*(double(weights[1])))/double(weights[2])
plt.plot(X,Y)
plt.show()
蓝色直线就是分类超平面，可以看到该直线按照红色和蓝色样本点的分布，将训练集分为两个部分。
(2)对测试集进行分类
# -*- coding: utf-8 -*-
import os
import sys
import numpy as np
import operator
from numpy import *
from common_libs import *
weights = mat([[ 4.12414349],[ 0.48007329],[-0.6168482 ]])
testdata = mat([-0.147324,2.874846])
m,n = shape(testdata)
testmat = zeros((m,n+1))
testmat[:,0] = 1
testmat[:,1:] = testdata
print classifier(testmat,weights)
三.算法分析
(1)超平面分析:超平面的斜率和截距，评估结果显示斜率参数已经达到平稳，但是截距并未达到平稳。
(2)权重向量的收敛评估
四.随机梯度下降法:算法改进与评估(引进随机样本抽取方式，并且提供了一种动态步长取值的策略)，既要保证优化精度，又要满足收敛速度。
(1)权重收敛评估
# -*- coding: utf-8 -*-
import os
import sys
import numpy as np
import operator
from numpy import *
from common_libs import *
import matplotlib.pyplot as plt
Input = file2matrix("testSet.txt","\t")
target = Input[:,-1] #获取分类标签列表
[m,n] = shape(Input)
dataMat = buildMat(Input)
# 4. 定义迭代次数
steps = 500 # 迭代次数
weights = ones(n) # 初始化权重向量
weightlist =[]
# 算法主程序:
# 1.对数据集的每个行向量进行m次随机抽取
# 2.对抽取之后的行向量应用动态步长
# 3.进行梯度计算
# 4.求得行向量的权值，合并为矩阵的权值
for j in xrange(steps):
dataIndex = range(m) # 以导入数据的行数m为个数产生索引向量:0~99
for i in xrange(m):
alpha = 2/(1.0+j+i)+0.0001 #动态修改alpha步长从4->0.016
randIndex = int(random.uniform(0,len(dataIndex))) #生成0~m之间随机索引
vectSum = sum(dataMat[randIndex]*weights.T) # 计算dataMat随机索引与权重的点积和
grad = logistic(vectSum) # 计算点积和的梯度
errors = target[randIndex]-grad # 计算误差
weights = weights + alpha * errors * dataMat[randIndex] #计算行向量权重
del(dataIndex[randIndex]) #从数据集中删除选取的随机索引
weightlist.append(weights)
lenwl= len(weightlist);
weightmat = zeros((lenwl,n))
i=0
for weight in weightlist:
weightmat[i,:]=weight
i+= 1
fig = plt.figure()
axes1 = plt.subplot(211); axes2 = plt.subplot(212)
X1 = np.linspace(0,lenwl,lenwl)
axes1.plot(X1,-weightmat[:,0]/weightmat[:,2]); # 截距
axes1.set_ylabel('Intercept')
axes2.plot(X1,-weightmat[:,1]/weightmat[:,2]); # 斜率
axes2.set_ylabel('Slope')
# 生成回归线
ws=standRegres(X1,-weightmat[:,0]/weightmat[:,2])
Y1=ws[0,0]+X1*ws[1,0]
axes1.plot(X1,Y1,color = 'red', linewidth=2, linestyle="-");
plt.show()
(2)权重分量的变化趋势
# -*- coding: utf-8 -*-
import os
import sys
import numpy as np
import operator
from numpy import *
from common_libs import *
import matplotlib.pyplot as plt
Input = file2matrix("testSet.txt","\t")
target = Input[:,-1] #获取分类标签列表
[m,n] = shape(Input)
dataMat = buildMat(Input)
# 4. 定义迭代次数
steps = 500 # 迭代次数
weights = ones(n) # 初始化权重向量
weightlist =[]
# 算法主程序:
# 1.对数据集的每个行向量进行m次随机抽取
# 2.对抽取之后的行向量应用动态步长
# 3.进行梯度计算
# 4.求得行向量的权值，合并为矩阵的权值
for j in xrange(steps):
dataIndex = range(m) # 以导入数据的行数m为个数产生索引向量:0~99
for i in xrange(m):
alpha = 2/(1.0+j+i)+0.0001 #动态修改alpha步长从4->0.016
randIndex = int(random.uniform(0,len(dataIndex))) #生成0~m之间随机索引
vectSum = sum(dataMat[randIndex]*weights.T) # 计算dataMat随机索引与权重的点积和
grad = logistic(vectSum) # 计算点积和的梯度
errors = target[randIndex]-grad # 计算误差
weights = weights + alpha * errors * dataMat[randIndex] #计算行向量权重
del(dataIndex[randIndex]) #从数据集中删除选取的随机索引
weightlist.append(weights)
lenwl= len(weightlist);
weightmat = zeros((lenwl,n))
i=0
for weight in weightlist:
weightmat[i,:]=weight
i+= 1
fig = plt.figure()
axes1 = plt.subplot(311); axes2 = plt.subplot(312); axes3 = plt.subplot(313)
X1 = np.linspace(0,lenwl,lenwl)
axes1.plot(X1,weightmat[:,0]); #
axes1.set_ylabel('weight[0]')
axes2.plot(X1,weightmat[:,1]); #
axes2.set_ylabel('weight[1]')
axes3.plot(X1,weightmat[:,2]); #
axes3.set_ylabel('weight[2]')
plt.show()
总结:本章主要讲述了:最优化方法、凸优化理论、计算复杂性、梯度计算、迭代寻优等机器学习的相关理论。
315
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
