回归2_机器学习

最新推荐文章于 2021-10-11 21:31:55 发布

明朝百晓生

最新推荐文章于 2021-10-11 21:31:55 发布

阅读量372

点赞数

分类专栏：人工智能

4AM_明朝百晓生

本文链接：https://blog.csdn.net/chengxf2/article/details/103369270

版权

人工智能专栏收录该内容

102 篇文章 3 订阅

订阅专栏

前言：

岭回归
lasso

一岭回归

1.1 函数模型：

$y_i=\beta x_i$

1.2 损失函数定义:

$J(\beta)=\left ( y \right -X \beta )^T \left ( y \right -X \beta )+ \lambda \beta^T \beta$

1.3 求偏导数：

$-X^Ty- X^Ty+2X^TX\beta +2\lambda \beta=0$

$\beta=(X^TX+\lambda I)^{-1}X^Ty$

因为：

$X^TX+\lambda I$ 行列式>0 ，所以可逆

1.4 代码实现

# -*- coding: utf-8 -*-
"""
Created on Tue Dec  3 16:33:37 2019

@author: chengxf2
"""

import numpy as np
import os
from sklearn import datasets
from sklearn.preprocessing import StandardScaler
import matplotlib.pyplot  as plt

"""
生成测试数据集
Arg
    None
return
    train_data: 训练样本
    train_label: 标签
    test_data:  测试集
    test_data:  测试集 标签
    
"""
def LoadDataSet():
    
    train_data = []
    train_label =[]
    
    test_data = []
    test_label =[]
    
    m = 5000
    n = 3
    m1 = 1000
    
    noseRate = 0.05 ##10%
    
    for i in range(m):
        x = np.random.rand(n)
        
        a = x[0]
        b = x[1]
        c=  x[2]
        
        y = 2.0+a+b*0.5+c*2.0
        if i<noseRate*m: ##加入5%的噪声数据
            y += np.random.rand(1)
       
        train_data.append([1.0,a,b,c])
        train_label.append(y)

    for j in range(m1):
        x1 = np.random.rand(n)
        a =x1[0];b=x1[1];c=x1[2]
        y1 = 2.0+a+b*0.5+c*2.0
        test_data.append([1.0,x1[0],x1[1],x1[2]])
        test_label.append(y1)
    return train_data, train_label, test_data,test_label
    

"""
计算回归系数
Args
   xMat: 输入数据集
   yMat: 标签集
   n: 样本维度
   lam: 系数
return 
   wMat: 系数
"""
def RidgeRegres(xMat, yMat,n, lam = 0.2):
    
   
    
    A = xMat.T*xMat+np.eye(n)*lam
    
    if np.linalg.det(A) ==0.0:
        return
    
    ws = np.linalg.inv(A)*(xMat.T*yMat)
    return ws
    
    
"""
岭回归
Args
   xArr: 训练数据集
   yArr: 训练标签集

return
   Lambdas: 使用的lamb
   wMat:  权值系数
"""   
def Ridge(xArr, yArr):
    
    xMat = np.mat(xArr)
    yMat = np.mat(yArr).T
    


    m,n = np.shape(xMat)
    print("\n ****Train Num*** \n   ",m)
 
    Lambdas=np.logspace(-4,4,100, base=2.71828182)
    #print("\n logspace ",Lambdas)
    num = len(Lambdas)
    wMat = np.zeros((num,n))

    i = 0
    for lamb in Lambdas:
        ws = RidgeRegres(xMat, yMat, n,lamb)
        wMat[i,:]=ws.T
        i=i+1
        
    #print("\n wMat: \n ",wMat)
    return np.mat(wMat),Lambdas


    
"""
验证
Args
  xArr: 测试集
  yArr: 输入数据集
   wMat: 权值系数

return
    Y： loss
    x:  lamb

"""
def Test(xArr ,yArr,wMat,Lambdas):
    
    xMat = np.mat(xArr)
    yMat = np.mat(yArr).T
    m,n = np.shape(wMat)
    num = len(xMat)
    x = np.arange(0,num,1.0)
    print("\n *******Test Num********** \n",num)
    

    

    
    for i in range(m):
        w = wMat[i,:].T
        lamb = Lambdas[i]
        yPred = xArr*w
        loss = np.sum(np.power(yPred-yMat,2))
        #print("\n \t lamb: ",lamb,"\t loss: ",loss)
        plt.scatter(lamb,loss,c='r')
    
    plt.xlabel("lamb")
    plt.ylabel("loss")
    plt.show() 
    
        
    

train_data, train_label, test_data,test_label = LoadDataSet()
wMat, Lambdas = Ridge(train_data, train_label)
Test(test_data, test_label, wMat, Lambdas)

说明：

噪声比例可以通过调整 noseRate 。当样本中噪声比例不同的时候，选用不同的 $\lambda$ ,会差异很大，

可以选择泛化性能最佳的一组

二 Lasso

相当于L1正规化。

适用于维度特别高，可以达到特征值稀疏化，降维效果

主要思路固定n-1项，求当前维度的驻点

损失函数

$J(\beta)=\sum (y-X\beta)^2+\lambda||\beta||_1$

$=RSS(\beta)+\lambda||\beta||_1$

对 RSS求 $\beta_j$ 偏导数

$\frac{RSS}{\beta_j}=-2 \sum_{i=1}^{m}x_{ij}(y_i-\sum_{j=1}^{n}\beta_jx_{ij})$

$=-2\sum_{i=1}^{m}x_{ij}(y_i-\sum_{k \neq j}^{m}\beta_kx_{ik}-\beta_j x_{ij})$

$=-2\sum_{i=1}^{m}x_{ij}(y_i-\sum_{k\neq j}^{m}\beta_jx_{ik})+2\beta_j\sum_{i=1} x_{ij}^2$

$=-2p_j+2\beta_j z_j$

对后面绝对值求导

è¿éåå¾çæè¿°

$\lambda |\beta_j|=\left\{\begin{matrix} \lambda, \beta_j >0\\ [-\lambda,\lambda], \beta_j=0 \\ -\lambda, \beta_j <0 \end{matrix}\right.$

二者相加

$\left\{\begin{matrix} -2p_j+2\beta_jz_j+\lambda=0, \beta_j>0\\ [-2p_j-\lambda, -2p_j+\lambda],\beta_j=0 \\ -2p_j+2\beta_jz_j-\lambda=0,\beta_j<0 \end{matrix}\right.$

则

$\beta_j=\left\{\begin{matrix} \frac{2p_j-\lambda}{2z_j}, \beta_j>0, p_j>\lambda/2\\ 0, p_j \in [-\lambda/2, \lambda/2] \\ \frac{\lambda+2p_j}{2z_j},p_j <\frac{-\lambda}{2} \end{matrix}\right.$

算法流程：

预置条件：

输入

m个样本

$Z=\sum_{i=1}^{m}x_{ij}^2$

流程

While(迭代开始）

for j in range(n) ##n 维度

计算 p_j

计算 $\beta_j$

当前 $\beta$ 变化范围或者迭代次数超过预置条件停止迭代

选用数据集2 结果如下

选用波士顿房价数据集结果如下：

代码实现流程：

# -*- coding: utf-8 -*-
"""
Created on Mon Dec  9 10:11:54 2019

@author: chengxf2

"""
import numpy as np
from sklearn.datasets import load_boston
import copy
import matplotlib.pyplot  as plt
class Lasso:
    
    """
    加载数据集
    Args:
        None
        函数模型y = x+5*x1+2*x2
    return 
        None
    """
    def LoadData2(self):
        
        m = 2000
        n = 13
        data = []
        target = []
        num = 1000
        
        for i in range(m):
            X = np.random.rand(n)
            data.append(X)
            y = X[0]*1.0+X[1]*5.0+X[2]*2.0
            target.append(y)
            #print("\n y",y)
        
        self.m,self.n = np.shape(data)
        
        self.trainData = np.mat(data[0:num])
        self.trainLabel = np.mat(target[0:num]).T
        self.testData = np.mat(data[num:-1])
        self.testLabel= np.mat(target[num:-1]).T
        
     
        
        Z = np.zeros((1,self.n))
        XX= np.power(self.trainData,2)
       # print("\n XX ",np.shape(XX))
        for j in range(self.n):
            Z[0,j]= np.sum(XX[:,j])
        #print("\n Z ",Z)
        self.Z = Z
            
            
    
    """
    加载波士顿房价数据集
    Args
       None
    return
       None
    """
    def  loadData(self):
        
        print("\n _______________________")
        boston = load_boston()
        num = 300 ###300个用于train
        data = boston.data
        target = boston.target
        self.m,self.n = np.shape(data)
        
        self.trainData = np.mat(data[0:num])
        self.trainLabel = np.mat(target[0:num]).T
        self.testData = np.mat(data[num:-1])
        self.testLabel= np.mat(target[num:-1]).T
        
     
        
        Z = np.zeros((1,self.n))
        XX= np.power(self.trainData,2)
       # print("\n XX ",np.shape(XX))
        for j in range(self.n):
            Z[0,j]= np.sum(XX[:,j])
        #print("\n Z ",Z)
        self.Z = Z
    
    """
    初始化
    """
    def __init__(self):
        
        self.m = 0 #样本个数
        self.n = 0 ##样本维度

        self.maxIter = 2000 ##最大迭代次数
        self.minE = 1e-6 ##最小误差
        self.Lambda = 2.0
        self.loadData()
        #self.LoadData2()
        
    
    """
    更新当前维度的权值系数
    Args
       pj:
       j: 当前的维度
    """
    def GetBeta(self, pj,j):
        z = 2.0*self.Z[0,j]
        
       # print("\n z",z)
        if pj>self.Lambda/2.0:
            
            b = (2*pj-self.Lambda)/z
        
        elif pj<-self.Lambda/2.0:
            b = (2*pj+self.Lambda)/z
            
        else:
            b = 0.0
        
        #print("\n b ",b)
        return b
        
        
        
        
    """
    训练
    """
    def Train(self):
        
        iter = 0
        W = np.mat(np.ones((self.n, 1)))
        
        oldW = None
       # print("W ",W[2][0])
       
       

        while iter<self.maxIter:
            oldW = copy.deepcopy(W)
            for j in range(self.n):
                
                xij = self.trainData[:,j]  ##(300,1)
                S= self.trainData*W- self.trainData[:,j]*W[j,0]  ##(300,1)
                S= self.trainLabel-S
                pj = sum(np.multiply(xij, S))  ##(300,13)
                #print("\n PJ ",pj, "\t S ",np.shape(S), "\t xij ",np.shape(xij))
                
                b = self.GetBeta(pj,j)
                
                W[j,0]= b
            
            diff = sum(np.power(oldW-W,2))
            #print("\n diff ",diff, "\t iter ",iter)
            iter =iter+1
            if diff<self.minE:
                break
        
        
        print("\n 权值系数: \n ",W, "\t iter ",iter)
        return W
   
    
    """
    预测
    """
    def Test(self, xMat, yMat, W):
        
        Y = xMat*W
        
        m,n = np.shape(xMat)
        
        for j in range(n):
            x= j
            y = abs(W[j,0])
            plt.scatter(x,y,c='r')
        
        plt.show()
        
        loss = 0.0
        for i in range(m):
            a = Y[i,0]-yMat[i,0]
            loss = loss+abs(a)
        print("\n loss: ",loss/m)
            
        
        
lasso = Lasso()     
W = lasso.Train()  
lasso.Test(lasso.testData,lasso.testLabel, W)

参考文档：

《机器学习实战》

https://blog.csdn.net/weixin_43374551/article/details/83688913