机器学习之逻辑斯蒂回归(logistic regression)
一、理论学习
- 模型
前面一篇博客学习线性回归,其实逻辑回归就是在线性回归基础上加了一个sigmoid激活函数。线性回归问题标签为连续性实数值,此为回归,而逻辑回归要解决的二分类问题,标签为离散的数值(0和1),因此线性回归计算的值加上sigmoid激活就将应变量映射到0到1之间,若大于0.5则为正例,反之则为负例。
前面线性回归博文中提到特征向量X与Y的一阶线性关系表示如下:
简写为:
加上sigmoid激活则为:
标签为1 的正样本和标签为0的负样本的概率表达式为:
1:
0:
2 、 策略
目标是找到合适参数W使得对于样本集中的样本满足映射关系,及模型对于输入的样本能正确预测正负样本。其实学习的是一个概率密度函数。损失函数(代价函数):
其中
看到这个公式就应该知道这是极大使然估计取负log后的表达式。目的让所有样本数据带入后让上面的式子最小,即让损失函数的值最小化。
3、算法
利用梯度下降法求解参数w使得损失函数最小化,对损失函数求偏导并令其为0,则可以推导得到其梯度表达式:
梯度更新式
二、实现
1、numpy实现
#coding=utf-8
# 文字编码格式的定义
import numpy as np
def sigmod(x):
return 1.0/(1.0+np.exp(-x))
def loadData(path):
dataMat=[] #特征
labelMat=[] #标签
fr = open(path) #打开txt文件
for line in fr.readlines():
lineArr = line.strip().split()
dataMat.append([1.0, float(lineArr[0]), float(lineArr[1])]) # 前面的1,表示方程的常量。比如两个特征X1,X2,共需要三个参数,W1+W2*X1+W3*X2
labelMat.append(int(lineArr[2]))
return dataMat, labelMat
def gradAscent(dataMatIn, classLabels,alpha,maxCycles):
dataMatrix = np.mat(dataMatIn) #convert to NumPy matrix
labelMat = np.mat(classLabels).transpose() #convert to NumPy matrix
m,n = np.shape(dataMatrix)
# alpha = 0.1
# maxCycles = 500
weights = np.ones((n,1))
for k in range(maxCycles): #heavy on matrix operations
h = sigmod(dataMatrix*weights) #matrix mult
error = (labelMat - h) #vector subtraction
weights = weights + alpha *1.0/m* dataMatrix.transpose()* error #matrix mult
return weights
if __name__ == '__main__':
path='E:/Machine learning/python_Learn/testSet.txt'
dataSet,labelMat=loadData(path)
weights=gradAscent(dataSet,labelMat,0.1,888)
testData=open(path)
testNum=0
rightNum=0
for line in testData.readlines():
testNum+= 1.0
lineArr = line.strip().split()
lineA=[]
lineA.append([1.0, float(lineArr[0]), float(lineArr[1])])
result=classifyVector(lineA,weights)
if result==float(lineArr[2]):
rightNum+=1.0
print('The Accurracy is '+(rightNum/testNum).__str__())
result=classifyVector(lineA,weights)
if result==float(lineArr[2]):
rightNum+=1.0
print('The Accurracy is '+(rightNum/testNum).__str__())
2、tensorflow实现
import tensorflow as tf
import numpy as np
import os
def loadData(path):
dataMat=[] #特征
labelMat=[] #标签
fr = open(path) #打开txt文件
for line in fr.readlines():
lineArr = line.strip().split()
dataMat.append([float(lineArr[0]), float(lineArr[1])]) # 前面的1,表示方程的常量。比如两个特征X1,X2,共需要三个参数,W1+W2*X1+W3*X2
labelMat.append(int(lineArr[2]))
return dataMat, labelMat
if __name__ == '__main__':
path='E:/Machine learning/python_Learn/testSet.txt'
dataSet,labelMat=loadData(path)
x=np.array(dataSet)
y=np.array(labelMat)
y = np.reshape(y, (100, 1))
#tensorflow
#定义数据入口
inputx=tf.placeholder(dtype=tf.float32,shape=[None,2])
gy=tf.placeholder(dtype=tf.float32,shape=[None,1])
#权重
w=tf.Variable(tf.random_normal([2,1]))
b=tf.Variable(tf.random_normal([1]))
#前向inference
with tf.device('/cpu:0'):
pred=tf.sigmoid(tf.add(tf.matmul(inputx,w),b))
#损失
loss= -1.0/100*tf.reduce_sum(gy*tf.log(pred) + (1 - gy) * tf.log(1 - pred))
#训练优化
train_op=tf.train.GradientDescentOptimizer(0.001).minimize(loss)
acc= tf.round(pred)
predictions_correct = tf.cast(tf.equal(acc, gy), tf.float32)
accuracy = tf.reduce_mean(predictions_correct)
tf.summary.scalar("loss",loss)
merged = tf.summary.merge_all()
init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
for i in range(60000):
_, prednp,lossnp,acc_np=sess.run([train_op,pred,loss,accuracy],feed_dict={inputx:x,gy:y})
if i%10==0:
print(i,lossnp,acc_np)
3、pytorch实现
import numpy as np
import os
import matplotlib.pyplot as plt
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.autograd import Variable
def loadData(path):
dataMat=[] #特征
labelMat=[] #标签
fr = open(path) #打开txt文件
for line in fr.readlines():
lineArr = line.strip().split()
dataMat.append([float(lineArr[0]), float(lineArr[1])]) # 前面的1,表示方程的常量。比如两个特征X1,X2,共需要三个参数,W1+W2*X1+W3*X2
labelMat.append(int(lineArr[2]))
return dataMat, labelMat
class Model(torch.nn.Module):
def __init__(self):
super(Model, self).__init__()
self.liner=nn.Linear(2,1)
self.sm=nn.Sigmoid()
def forward(self,x):
pred=self.liner(x)
pred=self.sm(pred)
return pred
if __name__ == '__main__':
path='E:/Machine learning/python_Learn/testSet.txt'
dataSet,labelMat=loadData(path)
data=np.array(dataSet)
labels=np.array(labelMat)
labels = np.reshape(labels, (100, 1))
x_data = (torch.from_numpy(data)).float()
y_data = (torch.from_numpy(labels)).float()
model = Model()
criterion = nn.BCELoss() # Defined loss function
optimizer = optim.SGD(model.parameters(), lr=0.1)
for epoch in range(800):
x_data=Variable(x_data)
y_data=Variable(y_data)
pred = model(x_data)
loss = criterion(pred, y_data)
optimizer.zero_grad()
loss.backward()
optimizer.step()
print(epoch, loss.item())
if epoch%10==0:
mask = pred.ge(0.5).float() # 以0.5为阈值进行分类
correct = (mask == y_data).sum() # 计算正确预测的样本个数
acc = correct.item() / x_data.size(0) # 计算精度
print("accuaracy is : ",acc,"%")