Stacking基本思想与简单实现

Stacking模型的基本思想

假设有1000条训练集，100条测试集，那么把训练集分为5份（一般分为5份），每一份有200条。用model训练其中四份，即800条，后，预测剩下200条，同时也预测测试集100条，得到预测结果。经过5次训练，训练集正好得到200×5条结果，也就是原来训练集的数量，合为一列，即1000×1的矩阵，测试集得到100×5条，将5次预测结果取平均值，得到100×1的矩阵，第一层任务结束。接着用相同的方法，尝试另外的模型，把不同模型得到的结果按列合并，若使用3个基模型，即得到1000×3的矩阵和100×3的矩阵，将这些结果作为第二层模型的训练集和测试集，初始的训练集标签作为第二层训练集标签，投入训练，预测结果。

简单实现

import numpy as np
from sklearn.model_selection import KFold
import pandas as pd
import warnings
warnings.filterwarnings('ignore')

# 创建一个父类，实现交叉训练的方法
class BasicModel(object):
    def train(self, x_train, y_train, x_val, y_val):
        pass

    def predict(self, model, x_test):
        pass

    def mode(slef,nums):
        num_dict = {}
        for i in nums:
            if i in num_dict:
                num_dict[i] += 1
            else:
                num_dict[i] = 1
        return max(num_dict.items(), key=lambda x: x[1])[0]

    def get_oof(self, x_train, y_train, x_test, n_folds=5):
        num_train, num_test = x_train.shape[0], x_test.shape[0]  # 读取矩阵第一维度的长度
        oof_train = np.zeros((num_train,))
        oof_test =[]

        oof_test_all_fold = np.zeros((num_test, n_folds))
        aucs = []
        KF = KFold(n_splits=n_folds, random_state=0)

        for i, (train_index, val_index) in enumerate(KF.split(x_train)): 
         # 得到原来训练集的4/5的训练集和1/5的测试集
            print('{0} fold, train {1}, val {2}'.format(i, len(train_index), len(val_index)))
            x_tra, y_tra = x_train[train_index], y_train[train_index]
            x_val, y_val = x_train[val_index], y_train[val_index]
            model, auc = self.train(x_tra, y_tra, x_val, y_val)  
            # 调用自身的train方法
            aucs.append(auc)
            oof_train[val_index] = self.predict(model, x_val)  
            # 得到第二层的训练集
            oof_test_all_fold[:, i] = self.predict(model, x_test)
'''
对于文本分类方面，最终得到地标签是整数，若取平均值会影响下一层判断，这里改进，求众数作为第二层模型测试集的输入
'''
        #找出众数算法：
        print('off_test_all_fold')
        print(oof_test_all_fold)
        for item in oof_test_all_fold:
            mode=self.mode(item)
            oof_test.append(mode)
        print(oof_test)

        print('all aucs {0}, average {1}'.format(aucs, np.mean(aucs)))
        return oof_train, oof_test


# 多项式朴素贝叶斯
from sklearn.naive_bayes import MultinomialNB as mnb
class MNBClassifier(BasicModel):
    def __init__(self):
        self.params = {
            'alpha': 1.0
        }

    def train(self, x_train, y_train, x_val, y_val):
        print('train with mnb model')
        model = mnb()
        model.fit(x_train,y_train)
        score = model.score(x_val, y_val)
        return model, score

    def predict(self, model, x_test):
        print('test with mnb model')
        # print(model.predict(x_test))
        return model.predict(x_test)


# 逻辑回归
from sklearn.linear_model import LogisticRegression as lgr
class LGRClassifier(BasicModel):
    def __init__(self):
        self.num_rounds = 1000
        self.early_stopping_rounds = 15

    def train(self, x_train, y_train, x_val, y_val):
        print('train with lgr model')
        model=lgr()
        model.fit(self.params,x_train, y_train)
        score = model.score(x_val, y_val)
        return model, score

    def predict(self, model, x_test):
        print('test with lgr model')
        # print(model.predict(x_test))
        return model.predict(x_test)

#支持向量机
from sklearn.svm import SVC
class SVCClassifier(BasicModel):
    def train(self, x_train, y_train, x_val, y_val):
        print('train with svc model')
        model = SVC()
        model.fit(x_train, y_train)
        score = model.score(x_val, y_val)
        return model, score

    def predict(self, model, x_test):
        print('test with svc model')
        print(model.predict(x_test))
        return model.predict(x_test)

def doJob(x_train, y_train, x_test, testLabel):
# 基模型处理
    mnb_classifier = MNBClassifier()
    mnb_oof_train, mnb_oof_test = mnb_classifier.get_oof(x_train, y_train, x_test)
    
    lgr_classifier = LGRClassifier()
    lgr_oof_train, lgr_oof_test = lgr_classifier.get_oof(x_train, y_train, x_test)
    
    svc_classifier = SVCClassifier()
    svc_oof_train, svc_oof_test = svc_classifier.get_oof(x_train, y_train, x_test)
    
    # 合并多个模型的结果
    input_train = [mnb_oof_train, lgr_oof_train,svc_oof_train]
    input_test = [mnb_oof_test, lgr_oof_test,svc_oof_test]
    input_test= np.array(input_test)

    stacked_train = np.concatenate([f.reshape(-1, 1) for f in input_train], axis=1)
    stacked_test = np.concatenate([f.reshape(-1, 1) for f in input_test], axis=1)
    
	#引入第二层模型
    from sklearn.linear_model import LinearRegression
    from  sklearn import metrics
    import lightgbm as lgb
    final_model = lgb.LGBMRegressor(objective='regression', num_leaves=31, learning_rate=0.05, n_estimators=20)

    final_model.fit(stacked_train, y_train)
    test_prediction = final_model.predict(stacked_test)
    print('test_prediction','\n',test_prediction)
    print(metrics.f1_score(test_prediction,testLabel,average='macro'))

参考链接：stacking基本思想与代码实现