混合专家系统(Mixture of experts)

MoE理论

参考：https://goker.wordpress.com/2011/07/01/mixture-of-experts/

实现代码

import numpy as np
import random
import matplotlib.pyplot as plt

class MOE:
    def __init__(self, train_x, train_y, k = 4, lamda = 0.1,inter = 50, lazy = 0.99, type = 'classification'):
        self._x = np.array(train_x)
        self._y = np.array(train_y)
        one = np.ones([len(train_x), 1])
        self._x = np.hstack([one, self._x])
        self._m, self._n = self._x.shape
        self._label_num = self._y.shape[1]


        self._k = k
        self._lamda = lamda
        self._inter = inter
        self._lazy = lazy
        self._type = type

        self._M = np.random.random_sample((self._k, self._n)) * 0.02 - 0.1
        self._V = np.random.random_sample((self._k, self._n, self._label_num)) * 0.02 - 0.1
        self._loglsoo_list = []
    def sgdTrain(self, test_x, test_y, ):
        self._test_x = np.array(test_x)
        self._test_y = np.array(test_y)
        one = np.ones([len(test_x), 1])
        self._test_x = np.hstack([one, self._test_x])

        iters = 0
        while(1):
            for t in xrange(self._m):                
                xt = self._x[t, :]
                rt = self._y[t, :]
                g = np.exp(np.dot(self._M, xt.transpose())).transpose()
                g = g / np.sum(g, axis=0)
                w = np.zeros((self._label_num, self._k))
                for j in xrange(self._label_num):
                    w[j, :] = np.dot(self._V[:,:,j],xt.transpose()).transpose()
                yi = np.dot(w, g.transpose()).transpose()

                ysi = np.exp(yi)
                ysi = ysi / np.sum(ysi)

                yih = np.exp(w)
                yih = yih / np.array([np.sum(yih, axis = 0)] * self._label_num)

                fh = np.exp(np.sum(np.log(yih) * np.array([rt] * self._k).transpose(), axis = 0)) * g
                fh = fh / np.sum(fh)

                for r in xrange(self._k):
                    for j in xrange(self._label_num):
                        dv = self._lamda * (rt[j] - yih[j, r]) * fh[r] * xt
                        self._V[r, :, j] = self._V[r, :, j] + dv
                    dm = self._lamda * (fh[r] - g[r]) * xt
                    self._M[r, :] = self._M[r, :] + dm
            self._lamda = self._lamda * self._lazy
            logloss = self.logloss()
            self._loglsoo_list.append(logloss)
            print "iters = ",iters, "logloss = ", logloss
            iters = iters + 1

            if iters > self._inter:
                break
    def ftrlTrain(self, test_x, test_y, alfa = 1.0, beta = 0.2, lamda1 = 0.0, lamda2 = 0.0):
        self._test_x = np.array(test_x)
        self._test_y = np.array(test_y)
        one = np.ones([len(test_x), 1])
        self._test_x = np.hstack([one, self._test_x])

        self._alfa = alfa
        self._beta = beta
        self._lamda1 = lamda1
        self._lamda2 = lamda2

        z_v = np.zeros((self._k, self._n, self._label_num))
        z_m = np.zeros((self._k, self._n))
        n_v = np.zeros((self._k, self._n, self._label_num))
        n_m = np.zeros((self._k, self._n))

        iters = 0
        while(1):
            for t in xrange(self._m):                
                xt = self._x[t, :]
                rt = self._y[t, :]
                g = np.exp(np.dot(self._M, xt.transpose())).transpose()
                g = g / np.sum(g, axis=0)
                w = np.zeros((self._label_num, self._k))
                for j in xrange(self._label_num):
                    w[j, :] = np.dot(self._V[:,:,j],xt.transpose()).transpose()
                yi = np.dot(w, g.transpose()).transpose()

                ysi = np.exp(yi)
                ysi = ysi / np.sum(ysi)

                yih = np.exp(w)
                yih = yih / np.array([np.sum(yih, axis = 0)] * self._label_num)

                fh = np.exp(np.sum(np.log(yih) * np.array([rt] * self._k).transpose(), axis = 0)) * g
                fh = fh / np.sum(fh)

                for r in xrange(self._k):
                    for j in xrange(self._label_num):
                        dv = -(rt[j] - yih[j, r]) * fh[r] * xt
                        sigma_v = (np.sqrt(n_v[r, :, j] + dv * dv) - np.sqrt(n_v[r, :, j])) / self._alfa
                        z_v[r, :, j] = z_v[r, :, j] + dv - sigma_v * self._V[r, :, j]
                        n_v[r, :, j] = n_v[r, :, j] + dv * dv
                        #for q in xrange(self._n):
                            #if(abs(z_v[r, q, j]) > self._lamda1):
                                #self._V[r, q, j] = -1.0 / ((self._beta + np.sqrt(n_v[r, q, j])) / self._alfa + self._lamda2) * (z_v[r, q, j] - np.sign(z_v[r, q, j]) * self._lamda1)
                            #else:
                                #self._V[r, q, j] = 0.0
                        self._V[r, :, j] = -1.0 / ((self._beta + np.sqrt(n_v[r, :, j])) / self._alfa + self._lamda2) * (z_v[r, :, j] - np.sign(z_v[r, :, j]) * self._lamda1)
                        index = np.where(np.abs(z_v[r, :, j]) <= self._lamda1)
                        self._V[r, index, j] = 0
                    dm = -(fh[r] - g[r]) * xt
                    sigma_m = (np.sqrt(n_m[r, :] + dm * dm) - np.sqrt(n_m[r, :])) / self._alfa
                    z_m[r, :] = z_m[r, :] + dm - sigma_m * self._M[r, :]
                    n_m[r, :] = n_m[r, :] + dm * dm
                    #for q in xrange(self._n):
                        #if(abs(z_m[r, q]) > self._lamda1):
                            #self._M[r, q] = -1.0 / ((self._beta + np.sqrt(n_m[r, q])) / self._alfa + self._lamda2) * (z_m[r, q] - np.sign(z_m[r, q]) * self._lamda1)
                        #else:
                            #self._M[r, q] = 0
                    self._M[r, :] = -1.0 / ((self._beta + np.sqrt(n_m[r, :])) / self._alfa + self._lamda2) * (z_m[r, :] - np.sign(z_m[r, :]) * self._lamda1)
                    index = np.where(np.abs(z_m[r, :]) <= self._lamda1)
                    self._M[r, index] = 0

            logloss = self.logloss()
            self._loglsoo_list.append(logloss)
            print "iters = ",iters, "logloss = ", logloss
            iters = iters + 1

            if iters > self._inter:
                break
    def ftrlTrainUpdate(self, test_x, test_y, alfa = 1.0, beta = 0.2, lamda1 = 0.0, lamda2 = 0.0):
        self._test_x = np.array(test_x)
        self._test_y = np.array(test_y)
        one = np.ones([len(test_x), 1])
        self._test_x = np.hstack([one, self._test_x])

        self._alfa = alfa
        self._beta = beta
        self._lamda1 = lamda1
        self._lamda2 = lamda2
        z_v = np.zeros((self._k, self._n, self._label_num))
        z_m = np.zeros((self._k, self._n))
        n_v = np.zeros((self._k, self._n, self._label_num))
        n_m = np.zeros((self._k, self._n))
        iters = 0
        while(1):
            for t in xrange(self._m):                
                xt = self._x[t, :]
                rt = self._y[t, :]
                g = np.exp(np.dot(self._M, xt.transpose())).transpose()
                g = g / np.sum(g, axis=0)
                w = np.zeros((self._label_num, self._k))
                for j in xrange(self._label_num):
                    w[j, :] = np.dot(self._V[:,:,j],xt.transpose()).transpose()
                yi = np.dot(w, g.transpose()).transpose()

                ysi = np.exp(yi)
                ysi = ysi / np.sum(ysi)
                yih = np.exp(w)
                yih = yih / np.array([np.sum(yih, axis = 0)] * self._label_num)           
                fh = np.exp(np.sum(np.log(yih) * np.array([rt] * self._k).transpose(), axis = 0)) * g
                fh = fh / np.sum(fh)
                #=============update V using FTRL==============
                dv = -((np.array([rt] * self._k) - yih.transpose())* fh.reshape(self._k, 1)).reshape(self._k, 1,self._label_num) * \
                   np.array([np.array([xt] * self._label_num).transpose()] * self._k)
                sigma_v = (np.sqrt(n_v + dv * dv) - np.sqrt(n_v)) / self._alfa
                z_v = z_v + dv - sigma_v * self._V
                n_v = n_v + dv * dv

                self._V = -1.0 / ((self._beta + np.sqrt(n_v)) / self._alfa + self._lamda2) * (z_v - np.sign(z_v) * self._lamda1)
                index = np.where(np.abs(z_v) <= self._lamda1)
                self._V[index] = 0
                #=============update M using FTRL==============
                dm = -(np.array(fh) - np.array(g)).reshape(self._k, 1) * np.array([xt] * self._k)
                sigma_m = (np.sqrt(n_m + dm * dm) - np.sqrt(n_m)) / self._alfa
                z_m = z_m + dm - sigma_m * self._M
                n_m = n_m + dm * dm            
                self._M = -1.0 / ((self._beta + np.sqrt(n_m)) / self._alfa + self._lamda2) * (z_m - np.sign(z_m) * self._lamda1)
                index = np.where(np.abs(z_m) <= self._lamda1)
                self._M[index] = 0            
            logloss = self.logloss()
            self._loglsoo_list.append(logloss)
            print "iters = ",iters, "logloss = ", logloss
            iters = iters + 1
            if iters > self._inter:
                break
    def deci(self, xi):
        xi = xi.reshape(1, self._n)
        w = np.zeros((self._label_num, self._k)) 
        for j in xrange(self._label_num):
            w[j, :] = np.dot(self._V[:, :, j], xi.transpose())[:, 0]
        g = np.exp(np.dot(self._M, xi.transpose()))
        g = g / np.sum(g)
        y = np.dot(w, g)
        p = np.exp(y)
        p = p / np.sum(p)
        return p
    def logloss(self):
        m = self._test_x.shape[0]
        logloss = 0
        for i in xrange(m):
            yi = self._test_y[i, :]
            p = self.deci(self._test_x[i, :])[:, 0]
            logloss = logloss + np.log2(np.power(p[0], yi[0])) + np.log2(np.power(p[1], yi[1]))
        return -logloss / m
    def predict(self, test_x):
        one = np.ones([len(test_x), 1])
        test_x = np.hstack([one, test_x])
        p_list = []
        for xi in test_x:
            p = self.deci(xi)
            p_list.append(p[:, 0])
        p_list = np.array(p_list)
        m, n = test_x.shape
        r = np.zeros((m, self._label_num))
        index = p_list[:, 0] >= 0.5
        r[index, 0] = 1
        index = p_list[:, 1] >= 0.5
        r[index, 1] = 1
        return r
def data():
    train_x = []
    train_y = []
    test_x = []
    test_y = []
    for line in file('train_x.txt', 'r'):
        ele = line.strip().split('\t')
        t = [float(e) for e in ele]
        train_x.append(t)
    for line in file('train_y.txt', 'r'):
        ele = line.strip().split('\t')
        t = [int(e) for e in ele]
        train_y.append(t)   
    for line in file('test_x.txt', 'r'):
        ele = line.strip().split('\t')
        t = [float(e) for e in ele]
        test_x.append(t)
    for line in file('test_y.txt', 'r'):
        ele = line.strip().split('\t')
        t = [int(e) for e in ele]
        test_y.append(t)  
    return [np.array(train_x), np.array(train_y),np.array(test_x), np.array(test_y),]      
if __name__ == '__main__':
    train_x, train_y, test_x, test_y = data()
    K = 4
    I = 10
    #original sgd
    sgd_moe = MOE(train_x, train_y, k = K, inter = I)
    sgd_moe.sgdTrain(test_x, test_y)
    p_list = sgd_moe.predict(test_x)    
    fig = plt.figure(1)
    ax = fig.add_subplot(3,1,1)
    ax.plot(train_x[train_y[:, 0] == 1, 0], train_x[train_y[:, 0] == 1, 1], 'ro')
    plt.plot(train_x[train_y[:, 1] == 1, 0], train_x[train_y[:, 1] == 1, 1], 'b.')
    plt.legend(['positive', 'negtive'])
    plt.title('train data')
    ax = fig.add_subplot(3,1,2)
    ax.plot(test_x[p_list[:, 0] == 1, 0], test_x[p_list[:, 0] == 1, 1], 'ro')
    ax.plot(test_x[p_list[:, 1] == 1, 0], test_x[p_list[:, 1] == 1, 1], 'b.')
    ax.legend(['positive', 'negtive'])
    plt.title('original sgd prediction result')
    #ftrl
    ftrl_moe = MOE(train_x, train_y, k = K, inter = I)
    ftrl_moe.ftrlTrainUpdate(test_x, test_y)
    p_list = ftrl_moe.predict(test_x)    
    ax = fig.add_subplot(3,1,3)
    ax.plot(test_x[p_list[:, 0] == 1, 0], test_x[p_list[:, 0] == 1, 1], 'ro')
    ax.plot(test_x[p_list[:, 1] == 1, 0], test_x[p_list[:, 1] == 1, 1], 'b.')
    plt.legend(['positive', 'negtive'])
    plt.title('ftrl prediction result')
    fig = plt.figure(2)
    plt.plot(sgd_moe._loglsoo_list, '-r.')
    plt.plot(ftrl_moe._loglsoo_list, '-bo')
    plt.xlabel("run number")
    plt.ylabel("log loss")
    plt.legend(['sgd logloss', "ftrl logloss"])
    plt.show()

训练和测试数据

点此下载，程序中用到的训练数据和测试数据

结果

• 训练数据，用原生的SGD和FTRL预测的结果分别如下图
  
• 原生的SGD和FTRL的logloss对比分别如下图