逻辑回归
一、什么是逻辑回归?
- 逻辑回归假设样本服从伯努利分布,利用极大似然估计,运用梯度下降法进行求解,从而达到将样本二分类的目的。
- 求导得minJ(θ)
import numpy as np
import matplotlib.pyplot as plt
def sigmoid(t):
return 1/(1+np.exp(-t))
x = np.linspace(-10,10,500)
y = sigmoid(x)
plt.plot(x,y)
plt.show()
二、决策边界
import numpy as np
from sklearn.metrics import accuracy_score
class LogitsicRegression:
def __init__(self):
"""初始化Logistic Regression模型"""
self.coef_ = None
self.intercept_ = None
self._theta = None
def _sigmoid(t):
return 1/(1+np.exp(-t))
def fit_gd(self, X_train, y_train, eta=0.01, n_iters=1e4):
"""根据训练数据集X_train,y_train,使用梯度下降法训练Logistic Regression模型"""
assert X_train.shape[0] == y_train.shape[0], \
"the size of X_train must be equal to the size of y_train"
def J(theta, X_b, y):
y_hat = self._sigmoid(X_b.dot(theta))
return -np.sum((y*np.log(y_hat)+ (1-y)*np.log(1-y_hat)) / len(y)
def dJ(theta, X_b, y):
# res = np.empty(len(theta))
# res[0] = np.sum(X_b.dot(theta) - y)
# for i in range(1, len(theta)):
# res[i] = (X_b.dot(theta) - y).dot(X_b[:, i])
# return res * 2 / len(X_b)
return X_b.T.dot(self._sigmoid(X_b.dot(theta)) - y) / len(X_b)
def gradient_descent(X_b, y, initial_theta, eta, n_iters=1e4, epsilon=1e-8):
theta = initial_theta
cur_iter = 0
while cur_iter < n_iters:
gradient = dJ(theta, X_b, y)
last_theta = theta
theta = theta - eta * gradient
if (abs(J(theta, X_b, y) - J(last_theta, X_b, y)) < epsilon):
break
cur_iter += 1
return theta
X_b = np.hstack([np.ones((len(X_train), 1)), X_train])
initial_theta = np.zeros(X_b.shape[1])
self._theta = gradient_descent(X_b, y_train, initial_theta, eta, n_iters)
self.intercept_ = self._theta[0]
self.coef_ = self._theta[1:]
return self
X_b = np.hstack([np.ones((len(X_train), 1)), X_train])
initial_theta = np.random.randn(X_b.shape[1])
self._theta = sgd(X_b, y_train, initial_theta, n_iters, t0, t1)
self.intercept_ = self._theta[0]
self.coef_ = self._theta[1:]
return self
def predict_proba(self, X_predict):
"""给定待预测数据集X_predict,返回表示X_predict的结果概率向量"""
assert self.intercept_ is not None and self.coef_ is not None, \
"must fit before predict!"
assert X_predict.shape[1] == len(self.coef_), \
"the feature number of X_predict must be equal to X_train"
X_b = np.hstack([np.ones((len(X_predict), 1)), X_predict])
return self._sigmoid(X_b.dot(self._theta))
def predict(self, X_predict):
"""给定待预测数据集X_predict,返回表示X_predict的结果概率向量"""
assert self.intercept_ is not None and self.coef_ is not None, \
"must fit before predict!"
assert X_predict.shape[1] == len(self.coef_), \
"the feature number of X_predict must be equal to X_train"
proba = self.predict_proba(X_predict)
return np.array(proba>=0.5,dtype='int')
def score(self, X_test, y_test):
"""根据测试数据集 X_test 和 y_test 确定当前模型的准确度"""
y_predict = self.predict(X_test)
return accuracy_score(y_test, y_predict)
def __repr__(self):
return "LogisticRegression()"
import numpy as np
import matplotlib.pyplot as plt
from sklearn import datasets
iris = datasets.load_iris()
X = iris.data
y = iris.target
X = X[y<2,:2]
y = y[y<2]
plt.scatter(X[y==0,0],X[y==0,1],color = 'red')
plt.scatter(X[y==1,0],X[y==1,1],color = 'blue')
plt.show()
from sklearn.model_selection import train_test_split
X_train,X_test,y_train,y_test = train_test_split(X,y,random_state=666)
from sklearn.linear_model import LogisticRegression
log_reg = LogisticRegression()
log_reg.fit(X_train,y_train)
def x2(x1):
return ((-log_reg.coef_[0])* x1 - log_reg.intercept_[0]) /log_reg.coef_[1]
- 不知道啥问题,运行不了该函数
x1_plot = np.linspace(4,8,1000)
x2_plot = x2(x1_plot)
plt.scatter(X[y==0,0],X[y==0,1],color = 'red')
plt.scatter(X[y==1,0],X[y==1,1],color = 'blue')
plt.plot(x1_plot,x2_plot)
plt.show()
plt.scatter(X_test[y_test==0,0],X_test[y_test==0,1],color = 'red')
plt.scatter(X_test[y_test==1,0],X_test[y_test==1,1],color = 'blue')
plt.plot(x1_plot,x2_plot)
plt.show()
三、多项式中的逻辑回归
import numpy as np
import matplotlib.pyplot as plt
np.random.seed(666)
X = np.random.normal(0,1,size=(200,2))
y = np.array(X[:,0]**2+ X[:,1]**2 <1.5,dtype='int')
plt.scatter(X[y==0,0],X[y==0,1])
plt.scatter(X[y==1,0],X[y==1,1])
plt.show()
from sklearn.linear_model import LogisticRegression
log_reg = LogisticRegression()
log_reg.fit(X,y)
log_reg.score(X,y)
- 使用逻辑回归
from sklearn.linear_model import LogisticRegression
log_reg = LogisticRegression()
log_reg.fit(X,y)
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import PolynomialFeatures,StandardScaler
def PolynomialFeaturesLogisticRegression(degree):
return Pipeline([
('poly',PolynomialFeatures(degree= degree)),
('std_scaler',StandardScaler()),
('log_reg',LogisticRegression())
])
- degree=20
四、逻辑回归中使用正则化
import numpy as np
import matplotlib.pyplot as plt
np.random.seed(666)
X = np.random.normal(0,1,size=(200,2))
y = np.array(X[:,0]**2+X[:,1]<1.5,dtype='int')
for _ in range(20):
y[np.random.randint(200)] = 1
plt.scatter(X[y==0,0],X[y==0,1])
plt.scatter(X[y==1,0],X[y==1,1])
plt.show()
from sklearn.model_selection import train_test_split
X_train,X_test,y_train,y_test = train_test_split(X,y,random_state=666)
from sklearn.linear_model import LogisticRegression
log_reg = LogisticRegression()
log_reg.fit(X_train,y_train)
# l2正则化
from sklearn.preprocessing import PolynomialFeatures,StandardScaler
from sklearn.pipeline import Pipeline
def PolynomialFeaturesLr(degree):
return Pipeline([
('poly',PolynomialFeatures(degree=degree)),
('std_scaler',StandardScaler()),
('log_reg',LogisticRegression())
])
poly_log_reg = PolynomialFeaturesLr(degree = 2)
poly_log_reg.fit(X_train,y_train)
4.1 使用C正则化?
def PolynomialFeaturesLrr(degree,C):
return Pipeline([
('poly',PolynomialFeatures(degree=degree)),
('std_scaler',StandardScaler()),
('log_reg',LogisticRegression(C=C))
])
poly_log_reg3 = PolynomialFeaturesLrr(degree=20,C=0.1)
poly_log_reg3.fit(X_train,y_train)
4.2 加入正则化
def PolynomialFeaturesLrrr(degree,C,penalty = 'l2'):
return Pipeline([
('poly',PolynomialFeatures(degree=degree)),
('std_scaler',StandardScaler()),
('log_reg',LogisticRegression(C=C,penalty=penalty))
])
poly_log_reg4 = PolynomialFeaturesLrrr(degree=20,C=0.1,penalty='l1')
poly_log_reg4.fit(X_train,y_train)
五、逻辑回归解决多分类问题
- OVR
- OVO
- MVM
西瓜书上有介绍
5.1 OVR
- 计算的复杂度提升了
5.2 OVO
每次只用真实的两个类别进行比较
import numpy as np
import matplotlib.pyplot as plt
from sklearn import datasets
iris = datasets.load_iris()
X = iris.data[:,:2]
y = iris.target
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
X_train,X_test,y_train,y_test = train_test_split(X,y,random_state=666)
log_reg = LogisticRegression()
log_reg.fit(X_train,y_train)
以前版本,multi_class=‘ovr’
- 绘制决策边界
绘制决策边界代码
def plot_decision_boundary(model, axis):
# meshgrid函数用两个坐标轴上的点在平面上画格,返回坐标矩阵
X0, X1 = np.meshgrid(
# 随机两组数,起始值和密度由坐标轴的起始值决定
np.linspace(axis[0], axis[1], int((axis[1] - axis[0]) * 100)).reshape(-1, 1),
np.linspace(axis[2], axis[3], int((axis[3] - axis[2]) * 100)).reshape(-1, 1),
)
# ravel()方法将高维数组降为一维数组,c_[]将两个数组以列的形式拼接起来,形成矩阵
X_grid_matrix = np.c_[X0.ravel(), X1.ravel()]
# 通过训练好的逻辑回归模型,预测平面上这些点的分类
y_predict = model.predict(X_grid_matrix)
y_predict_matrix = y_predict.reshape(X0.shape)
# 设置色彩表
from matplotlib.colors import ListedColormap
my_colormap = ListedColormap(['#0000CD', '#40E0D0', '#FFFF00'])
# 绘制等高线,并且填充等高区域的颜色
plt.contourf(X0, X1, y_predict_matrix, linewidth=5, cmap=my_colormap)
== OVO==
log_reg2 = LogisticRegression(multi_class='multinomial',solver='newton-cg')
log_reg2.fit(X_train,y_train)
log_reg2.score(X_test,y_test)
- 使用全部数据
iris = datasets.load_iris()
X = iris.data
y = iris.target
X_train,X_test,y_train,y_test = train_test_split(X,y,random_state=666)
log_reg = LogisticRegression()
log_reg.fit(X_train,y_train)
log_reg.score(X_train,y_train)
log_reg2 = LogisticRegression(multi_class='multinomial',solver='newton-cg')
log_reg2.fit(X_train,y_train)
log_reg2.score(X_train,y_train)
5.3 对所有的二分类器进行多分类任务
from sklearn.multiclass import OneVsOneClassifier,OneVsRestClassifier
ovr = OneVsRestClassifier(log_reg)
ovr.fit(X_train,y_train)
ovr.score(X_test,y_test)
ovo = OneVsOneClassifier(log_reg)
ovo.fit(X_train,y_train)
ovo.score(X_test,y_test)