机器学习之Softmax回归（Python实现）

• Softmax回归模型是logistic回归模型在多分类问题上的推广，在多分类问题中，类标签 y 可以取两个以上的值。
• Softmax回归模型对于诸如MNIST手写数字分类等问题是很有用的，该问题的目的是辨识10个不同的单个数字。
• Softmax回归是有监督的

import numpy as np  
import matplotlib.pylab as plt  
import copy  
from scipy.linalg import norm  
from math import pow  
from scipy.optimize import fminbound,minimize  
import random  
def _dot(a, b):  
    # 表示点乘
    mat_dot = np.dot(a, b)  
    # exp表示e的多次次幂
    return np.exp(mat_dot)  

def condProb(theta, thetai, xi):  
    # 计算样本i的输出
    numerator = _dot(thetai, xi.transpose())  

    # 计算所有样本的输出求和
    denominator = _dot(theta, xi.transpose())  
    denominator = np.sum(denominator, axis=0)  
    p = numerator / denominator  
    return p  

'''  下面就是求得代价函数 '''
def costFunc(alfa, *args):  
    i = args[2]  
    original_thetai = args[0]  
    delta_thetai = args[1]  
    x = args[3]  
    y = args[4]  
    lamta = args[5]  

    labels = set(y)  
    thetai = original_thetai  
    thetai[i, :] = thetai[i, :] - alfa * delta_thetai  
    k = 0  
    sum_log_p = 0.0  
    for label in labels:  
        index = y == label  
        xi = x[index]  
        p = condProb(original_thetai,thetai[k, :], xi)  
        log_p = np.log10(p)  
        sum_log_p = sum_log_p + log_p.sum()  
        k = k + 1  
    # 这就是代价函数的公式
    r = -sum_log_p / x.shape[0]+ (lamta / 2.0) * pow(norm(thetai),2)  
     #print r ,alfa  

    return r  
class Softmax:  
    def __init__(self, alfa, lamda, feature_num, label_mum, run_times = 500, col = 1e-6):  
        self.alfa = alfa  
        self.lamda = lamda  
        self.feature_num = feature_num  
        self.label_num = label_mum  
        self.run_times = run_times  
        self.col = col  
        self.theta = np.random.random((label_mum, feature_num + 1))+1.0  
    def oneDimSearch(self, original_thetai,delta_thetai,i,x,y ,lamta):  
         res = minimize(costFunc, 0.0, method = 'Powell', args =(original_thetai,delta_thetai,i,x,y ,lamta))  
         return res.x  

    def train(self, x, y):
         tmp = np.ones((x.shape[0], x.shape[1] + 1))
         tmp[:,1:tmp.shape[1]] = x
         x = tmp
         del tmp
         labels = set(y)
         self.errors = []
         old_alfa = self.alfa
         for kk in range(0, self.run_times):
             i = 0

             # 因为Softmax中不同标签的样本的权值是不一样的
             # 下面就是更新每个标签样本对应的权值
             for label in labels:
                 tmp_theta = copy.deepcopy(self.theta)
                 one = np.zeros(x.shape[0])
                 index = y == label
                 # one位x.shape[0]行的数组，将分类是对应的类别设置为1
                 one[index] =  1.0
                 thetai = np.array[self.theta[i,:]]
                 #  预测输出结果
                 prob = self.condProb(thetai, x)
                 # 标签与预测结果的差
                 prob = np.array([one - prob])
                 prob = prob.transpose()
                 # 此处为梯度表达式
                 delta_thetai=-np.sum(x*prob, axis=0)/x.shape[0]+self.lamda * self.theta[i, :]
                 #alfa = self.oneDimSearch(self.theta,delta_thetai,i,x,y ,self.lamda)#一维搜索法寻找最优的学习率，没有实现
                 # 更新权值                 
                 self.theta[i, :] = self.theta[i,:] - self.alfa*np.array([delta_thetai])
                 i = i + 1
             self.errors.append(self.performance(tmp_theta))

    def performance(self, tmp_theta):
         return norm(self.theta - tmp_theta)

    def dot(self, a, b):
         mat_dot = np.dot(a, b)
         return np.exp(mat_dot)

    def condProb(self, thetai, xi):
         numerator = self.dot(thetai, xi.transpose())

         denominator = self.dot(self.theta, xi.transpose())
         denominator = np.sum(denominator, axis = 0)
         p = numerator[0] / denominator
         return p

    def predict(self, x):
         tmp = np.ones((x.shape[0], x.shape[1]+1))
         tmp[:,1:tmp.shape[1]] = x
         x = tmp
         row = x.shape[0]
         col = self.theta.shape[0]
         pre_res = np.zeros((row, col))
         for i in range(0, row):
             xi = x[i, :]
             for j in range(0, col):
                 thetai = self.theta[j, :]
                 p = self.condProb(np.array([thetai]), np.array([xi]))
                 pre_res[i, j] = p
         r = []
         for i in range(0, row):
             tmp = []
             line = pre_res[i, :]
             ind = line.argmax()
             tmp.append(ind)
             tmp.append(line[ind])
             r.append(tmp)
         return np.array(r)

    def evaluate(self):
         pass

def sample(sample_num, feature_num, label_num):
    n = int(sample_num / label_num)
    x = np.zeros((n*label_num, feature_num))
    y = np.zeros(n*label_num, dtype = np.int)
    for i in range(0, label_num):
        x[i*n : i*n+n, :] = np.random.random((n,feature_num)) + i
        y[i*n : i*n+n] = i
    return [x,y]

def save(name, x, y):
    writer = open(name, 'w')
    for i in range(0, x.shape[0]):
        for j in range(0, x.shape[1]):
            writer.write(str(x[i,j]) + ' ')
        writer.write(str(y[i])+'\n')
    writer.close()

def load(name):
    x = []
    y = []
    for line in open(name, 'r'):
        ele = line.split(' ')
        tmp = []
        for i in range(0, len(ele) - 1):
            tmp.append(float(ele[i]))
        x.append(tmp)
        y.append(int(ele[len(ele) - 1]))
    return [x,y]

def plotRes(pre, real, test_x, l):
    s = set(pre)
    col = ['r', 'b', 'g', 'y', 'm']
    fig = plt.figure()

    ax = fig.add_subplot(111)
    for i in range(0, len(s)):
        index1 = pre == i
        index2 = real == i
        x1 = test_x[index1, :]
        x2 = test_x[index2, :]
        ax.scatter(x1[:,0], x1[:,1],color=col[i],marker='v',linewidths=0.5)
        ax.scatter(x2[:,0], x2[:,1],color=col[i],marker='.',linewidths=12)
    plt.title('learning rating='+str(1))
    plt.legend('c1:predict','c1:true',
               'c2:predict','c2:true',
               'c3:predict','c3:true',
               'c4:predict','c4:true',
               'c5:predict','c5:true'),shadow = true,loc = (0.01, 0.4)
    plt.show()

if __name__ == '__main__':
    # [x,y] = samples(1000,2,5)
    # save('data.txt', x, y)
    [x,y] = load('data.txt')
    index = range(0, len(x))
    random.shuffle(index)
    x = np.array(x)
    y = np.array(y)
    x_train = x[index[0:700],:]
    y_train = y[index[0:700]]
    #这里讲第二个参数设置为0.0，即不用正则化，因为模型中没有高次项，用正则化反而使效果变差  
    softmax = Softmax(0.4, 0.0, 2, 5)
    softmax.train(x_train, y_train)
    x_test = x[index[700:1000],:]
    y_test = y[index[700:1000]]
    r = softmax.predict(x_test)
    plotRes(r[:,0], y_test, x_test, softmax.alfa)
    t = r[:,0] != y_test
    o = np.zeros(len(t))
    o(t) = 1
    err = sum(o)

如果帮到你了，请赞赏支持：