DataWhale数据挖掘（二手车价格预测）第五次打卡

模型融合目标

1、对于多种调参完成的模型进行模型融合。
2、完成对于多种模型的融合，提交融合结果。

导入库和数据

import itertools
import matplotlib.gridspec as gridspec
from sklearn import datasets
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import cross_val_score,train_test_split

from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import train_test_split

from sklearn import linear_model
from sklearn import preprocessing
from sklearn.svm import SVR
from sklearn.decomposition import PCA,FastICA,FactorAnalysis,SparsePCA

import lightgbm as lgb
import xgboost as xgb
from sklearn.model_selection import GridSearchCV,cross_val_score
from sklearn.ensemble import RandomForestRegressor,GradientBoostingRegressor

from sklearn.metrics import mean_absolute_error,mean_squared_error

#读取数据
TrainData=pd.read_csv('used_car_train_20200313.csv',sep=' ')
TestData=pd.read_csv('used_car_testA_20200313.csv',sep=' ')
print(TrainData.shape,TestData.shape)

numerical_cols=TrainData.select_dtypes(exclude='object').columns
numerical_cols

feature_cols=[col for col in numerical_cols if col not in ['SaleID','name','regDate','price']]

X_data=TrainData[feature_cols]
Y_data=TrainData['price']

X_test=TestData[feature_cols]

print('x train shape:',X_data.shape)
print('x test shape:',X_test.shape)

数据缺失简单处理

def Sat_info(data):
    print("Sat_info:")
    print("mean:",np.mean(data))
    print("std:",np.std(data))
    print("min:",np.min(data))
    print("max:",np.max(data))
    print("ptp:",np.ptp(data))
    print("var:",np.var(data))

X_data=X_data.fillna(-1)
X_test=X_test.fillna(-1)

构建模型

def build_model_lr(x_train,y_train):
    reg_model=linear_model.LinearRegression()
    reg_model.fit(x_train,y_train)
    return reg_model

def build_model_ridge(x_train,y_train):
    reg_model=linear_model.Ridge(alpha=0.8)
    reg_model.fit(x_train,y_train)
    return reg_model

def build_model_lasso(x_train,y_train):
    reg_model=linear_model.Lasso()
    reg_model.fit(x_train,y_train)
    return reg_model

def build_model_gbdt(x_train,y_train):
    estimator=GradientBoostingRegressor(loss='ls',subsample=0.85,max_depth=5,n_estimators=100)
    param_grid={'learning_rate':[0.05,0.08,0.1,0.2]}
    gbdt=GridSearchCV(estimator,param_grid,cv=3)
    gbdt.fit(x_train,y_train)
    print("gbdt best param:",gbdt.best_params_)
    return gbdt

def build_model_xgb(x_train,y_train):
    model=xgb.XGBRegressor(n_estimators=120,learning_rate=0.08,gamma=0,subsample=0.8,
                          colsample_bytree=0.9,max_depth=5)
    model.fit(x_train,y_train)
    return model

def build_model_lgb(x_train,y_train):
    estimator=lgb.LGBMRegressor(num_leaves=63,n_estimators=100)
    param_grid={'learning_rate':[0.01,0.05,0.1]}
    lgb=GridSearchCV(estimator,param_grid)
    lgb.fit(x_train,y_train)
    return lgb

#五折交叉验证
xgr=xgb.XGBRegressor(n_estimators=120,learning_rate=0.1,subsample=0.8,colsample_bytree=0.9,max_depth=7)

scores_train=[]
scores=[]

## 5折交叉验证方式
sk=StratifiedKFold(n_splits=5,shuffle=True,random_state=0)
for train_ind,val_ind in sk.split(X_data,Y_data):
    
    train_x=X_data.iloc[train_ind].values
    train_y=Y_data.iloc[train_ind]
    val_x=X_data.iloc[val_ind].values
    val_y=Y_data.iloc[val_ind]
    
    xgr.fit(train_x,train_y)
    pred_train_xgb=xgr.predict(train_x)
    pred_xgb=xgr.predict(val_x)
    
    score_train = mean_absolute_error(train_y,pred_train_xgb)
    scores_train.append(score_train)
    score = mean_absolute_error(val_y,pred_xgb)
    scores.append(score)

print('Train mae:',np.mean(score_train))
print('Val mae',np.mean(scores))

#划分数据集，并用多种方法训练和测试
## Split data with val
x_train,x_val,y_train,y_val = train_test_split(X_data,Y_data,test_size=0.3)

## Train and Predict
print('Predict LR...')
model_lr = build_model_lr(x_train,y_train)
val_lr = model_lr.predict(x_val)
subA_lr = model_lr.predict(X_test)

print('Predict Ridge...')
model_ridge = build_model_ridge(x_train,y_train)
val_ridge = model_ridge.predict(x_val)
subA_ridge = model_ridge.predict(X_test)

print('Predict Lasso...')
model_lasso = build_model_lasso(x_train,y_train)
val_lasso = model_lasso.predict(x_val)
subA_lasso = model_lasso.predict(X_test)

print('Predict GBDT...')
model_gbdt = build_model_gbdt(x_train,y_train)
val_gbdt = model_gbdt.predict(x_val)
subA_gbdt = model_gbdt.predict(X_test)

#比赛常用算法
print('predict XGB...')
model_xgb = build_model_xgb(x_train,y_train)
val_xgb = model_xgb.predict(x_val)
subA_xgb = model_xgb.predict(X_test)

print('predict lgb...')
model_lgb = build_model_lgb(x_train,y_train)
val_lgb = model_lgb.predict(x_val)
subA_lgb = model_lgb.predict(X_test)

模型融合应用

#加权融合
def Weighted_method(test_pre1,test_pre2,test_pre3,w=[1/3,1/3,1/3]):
    Weighted_result = w[0]*pd.Series(test_pre1)+w[1]*pd.Series(test_pre2)+w[2]*pd.Series(test_pre3)
    return Weighted_result

## Init the Weight
w = [0.3,0.4,0.3]

## 测试验证集准确度
val_pre = Weighted_method(val_lgb,val_xgb,val_gbdt,w)
MAE_Weighted = mean_absolute_error(y_val,val_pre)
print('MAE of Weighted of val:',MAE_Weighted)

## 预测数据部分
subA = Weighted_method(subA_lgb,subA_xgb,subA_gbdt,w)
print('Sta inf:')
Sta_inf(subA)
## 生成提交文件
sub = pd.DataFrame()
sub['SaleID'] = X_test.index
sub['price'] = subA
sub.to_csv('./sub_Weighted.csv',index=False)

## 与简单的LR（线性回归）进行对比
val_lr_pred = model_lr.predict(x_val)
MAE_lr = mean_absolute_error(y_val,val_lr_pred)
print('MAE of lr:',MAE_lr)

#stacking融合

## 第一层
train_lgb_pred = model_lgb.predict(x_train)
train_xgb_pred = model_xgb.predict(x_train)
train_gbdt_pred = model_gbdt.predict(x_train)

Strak_X_train = pd.DataFrame()
Strak_X_train['Method_1'] = train_lgb_pred
Strak_X_train['Method_2'] = train_xgb_pred
Strak_X_train['Method_3'] = train_gbdt_pred

Strak_X_val = pd.DataFrame()
Strak_X_val['Method_1'] = val_lgb
Strak_X_val['Method_2'] = val_xgb
Strak_X_val['Method_3'] = val_gbdt

Strak_X_test = pd.DataFrame()
Strak_X_test['Method_1'] = subA_lgb
Strak_X_test['Method_2'] = subA_xgb
Strak_X_test['Method_3'] = subA_gbdt

## level2-method 
model_lr_Stacking = build_model_lr(Strak_X_train,y_train)
## 训练集
train_pre_Stacking = model_lr_Stacking.predict(Strak_X_train)
print('MAE of Stacking-LR:',mean_absolute_error(y_train,train_pre_Stacking))

## 验证集
val_pre_Stacking = model_lr_Stacking.predict(Strak_X_val)
print('MAE of Stacking-LR:',mean_absolute_error(y_val,val_pre_Stacking))

## 预测集
print('Predict Stacking-LR...')
subA_Stacking = model_lr_Stacking.predict(Strak_X_test)

subA_Stacking[subA_Stacking<10]=10  ## 去除过小的预测值

sub = pd.DataFrame()
sub['SaleID'] = TestA_data.SaleID
sub['price'] = subA_Stacking
sub.to_csv('./sub_Stacking.csv',index=False)

总结

一般使用模型融合主要的几种方式，具体使用看结果进行调整。