Task4 建模与调参

#读取数据
导入工具包

import pandas as pd
import numpy as np
import warnings
warnings.filterwarnings('ignore')

创建reduce_mem_usage函数通过调整数据类型，帮助我们减少数据在内存中占用的时间

def reduce_mem_usage(df):
    start_mem = df.memory_usage().sum()
    print("Memory usage of datagrame is {:2f} MB".format(start_mem))
    
    for col in df.columns:
        col_type = df[col].dtype
        
        if col_type != object:
            c_min = df[col].min() #去除列的最大值和最小值
            c_max = df[col].max()
            if str(col_type)[:3]  == 'int':
               #依次尝试转化为int8\int16\int32\int64，如果数据大小没溢出则转化
                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:
                    df[col] = df[col].astype(np.int8)
                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:
                    df[col] = df[col].astype(np.int16)
                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:
                    df[col] = df[col].astype(np.int32)
                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:
                    df[col] = df[col].astype(np.int64)
            else:   #不是整形则是浮点型
                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:
                    df[col] = df[col].astype(np.float16)
                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:
                    df[col] = df[col].astype(np.float32)
                else:
                    df[col] = df[col].astype(np.float64)
        else:
            df[col] = df[col].astype('category')
    
    end_mem = df.memory_usage().sum()
    print('Memory usage after optimization is: {:.2f}MB'.format(end_mem))
    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))
    return df

读取数据并压缩，可看出数据大小压缩了73.4%

sample_feature = reduce_mem_usage(pd.read_csv('data_for_tree.csv'))

Memory usage of datagrame is 62099672.000000 MB
Memory usage after optimization is: 16520303.00MB
Decreased by 73.4%

continuous_feature_names= [ x for x in sample_feature.columns if x not in ['price','brand','model']]

将需要用的特征挑出(仅列名,用来分离特征集和标签集)

#4.4.2 线性回归 & 五折交叉验证 % 模拟真实业务情况

sample_feature = sample_feature.dropna().replace('-',0).reset_index(drop=True)
sample_feature['notRepairedDamage'] = sample_feature['notRepairedDamage'].astype(np.float32)
train = sample_feature[continuous_feature_names + ['price']]

train_X = train[continuous_feature_names]
train_y = train['price'

处理缺失数据，构造训练集

##4.4.2 -1 简单建模

from sklearn.linear_model import LinearRegression
model = LinearRegression(normalize=True)
model = model.fit(train_X, train_y)

查看训练的线性回归模型的截距(intercept) 与 权重(coef)

'intercept:' + str(model.intercept_)
sorted(dict(zip(continuous_feature_names, model.coef_)).items(),key= lambda x:x[1],reverse=True)

from matplotlib import pyplot as plt
subsample_index = np.random.randint(low=0, high=len(train_y), size=50)

绘制特征v_9的值与标签的散点图

plt.scatter(train_X['v_9'][subsample_index], train_y[subsample_index], color='black')
plt.scatter(train_X['v_9'][subsample_index], model.predict(train_X.loc[subsample_index]),color='blue')
plt.xlabel('v_9')
plt.ylabel('price')
plt.legend(['Ture Price','Predicted Price'],loc='upper right')
print('The predicted price is obvious different from true price')
plt.show()

图片发现模型的预测结果（蓝点）与真实标签（黑点）的分布差异较大，切部分预测值出现了小于0的情况，说明我们的模型存在一些问题

import seaborn as sns
print('It is clear to see the price shows a typical exponential distribution')
plt.figure(figsize=(15,5))
plt.subplot(1,2,1)
sns.distplot(train_y)
plt.subplot(1,2,2)
sns.distplot(train_y[train_y < np.quantile(train_y, 0.9)])

观察价格分布图，数据的标签(price)呈现长尾分布。
很多模型都假设数据误差项符合正态分布\n而长尾分布的数据违背了这一假设。如果误差项不呈正态分布，意味着置信区间会变得很不稳定

train_y_ln = np.log(train_y + 1)

为更加贴近正态分布,对标签price进行log(x+1)变换

import seaborn as sns
print('The transformed price sees like normal distribution')
plt.figure(figsize=(15,5))
plt.subplot(1,2,1)
sns.distplot(train_y_ln)
plt.subplot(1,2,2)
sns.distplot(train_y_ln[train_y_ln < np.quantile(train_y_ln, 0.9)])

再训练一次，查看截距与权重

model = model.fit(train_X, train_y_ln)

print('intercept:' + str(model.intercept_))
sorted(dict(zip(continuous_feature_names, model.coef_)).items(), key=lambda x:x[1], reverse=True)

plt.scatter(train_X['v_9'][subsample_index], train_y[subsample_index], color='black')
plt.scatter(train_X['v_9'][subsample_index], np.exp(model.predict(train_X.loc[subsample_index])),color='blue')
plt.xlabel('v_9')
plt.ylabel('price')
plt.legend(['True Price','Predicted Price'],loc='upper right')
print('The predicted price seems normal after np.log transforming')
plt.show()

再次进行可视化，发现预测结果与真实值较为接近，且未出现异常状况

##4.4.2 - 2 五折交叉验证

from sklearn.model_selection import cross_val_score
from sklearn.metrics import mean_absolute_error, make_scorer

定义一个函数,用来处理预测值和真实值的log变换

def log_transfer(func):
    def wrapper(y, yhat):
        result = func(np.log(y), np.nan_to_num(np.log(yhat)))
        return result
    return wrapper

scores = cross_val_score(model, X=train_X, y=train_y, verbose=1,cv=5,
                         scoring=make_scorer(log_transfer(mean_absolute_error)))
print('AVG:', np .mean(scores))

使用线性回归模型，对未处理标签的特征数据进行五折交叉验证(Error 1.36)

scores = cross_val_score(model, X=train_X, y=train_y_ln, verbose=1, cv=5,
                         scoring=make_scorer(mean_absolute_error))
print('AVG:',np.mean(scores))

使用线性回归模型，对处理过标签的特征数据进行五折交叉验证(Error 0.19)

scores = pd.DataFrame(scores.reshape(1,-1))
scores.columns = ['cv' + str(x) for x in range(1,6)]
scores.index = ['MAE']
scores

事实上，由于我们并不具有预知未来的能力，五折交叉验证在某些与时间相关的数据集上反而反映了不真实的情况。通过2018年的二手车价格预测2017年的二手车价格，这显然是不合理的，因此我们还可以采用时间顺序对数据集进行分隔。
在本例中，我们选用靠前时间的4/5样本当作训练集，靠后时间的1/5当作验证集，最终结果与五折交叉验证差距不大

import datetime
sample_feature = sample_feature.reset_index(drop=True) #重置索引
split_point = len(sample_feature) // 5 * 4 #设置分割点
train = sample_feature.loc[:split_point].dropna()
val = sample_feature.loc[split_point:].dropna()

train_X = train[continuous_feature_names]
train_y_ln = np.log(train['price']+1)
val_X = val[continuous_feature_names]
val_y_ln = np.log(val['price'] + 1)
model = model.fit(train_X, train_y_ln)
mean_absolute_error(val_y_ln, model.predict(val_X))

MAE为0.196

##4.4.2 -4 绘制学习曲线与验证曲线

from sklearn.model_selection import learning_curve, validation_curve

def plot_learning_curve(estimator, title, X, y, ylim=None, cv=None,n_jobs=1, 
                        train_sizes=np.linspace(.1, 1.0,5)):
    plt.figure()
    plt.title(title)
    if ylim is not None:
        plt.ylim(*ylim)
    plt.xlabel('Training example')
    plt.ylabel('score')
    train_sizes, train_scores, test_scores = learning_curve(estimator, X, y, cv=cv, n_jobs=n_jobs,
                                                           train_sizes=train_sizes,
                                                           scoring=make_scorer(mean_absolute_error))
    train_scores_mean = np.mean(train_scores, axis=1)
    train_scores_std = np.std(train_scores, axis=1)
    test_scores_mean = np.mean(test_scores, axis=1)
    test_scores_std = np.std(test_scores, axis=1)
    plt.grid() #区域
    plt.fill_between(train_sizes, train_scores_mean - train_scores_std,
                    train_scores_mean + train_scores_std, alpha=0.1,
                    color="r")
    plt.fill_between(train_sizes, test_scores_mean - test_scores_std,
                    test_scores_mean + test_scores_std, alpha=0.1,
                    color="g")
    plt.plot(train_sizes, train_scores_mean, 'o-', color='r',label="Training score")
    plt.plot(train_sizes, test_scores_mean, 'o-', color="g",label="Cross_validation score")
    plt.legend(loc="best")
    return plt
    
plot_learning_curve(LinearRegression(), 'Liner_model', train_X[:1000], train_y_ln[:1000],
                   ylim=(0.0,0.5), cv=5, n_jobs=1)

一飞冲天了还没查出哪里不对

#4.4.3 多种模型对比

train = sample_feature[continuous_feature_names + ['price']].dropna()

train_X = train[continuous_feature_names]
train_y = train['price']
train_y_ln = np.log(train_y +1)

4.4.3 -1 线性模型 嵌入式特征选择

观察三种模型的效果对比

from sklearn.linear_model import LinearRegression
from sklearn.linear_model import Ridge
from sklearn.linear_model import Lasso

models = [LinearRegression(),
         Ridge(),
         Lasso()]
         
result = dict()
for model in models:
    model_name = str(model).split('(')[0]
    scores = cross_val_score(model, X=train_X, y=train_y_ln, verbose=0, cv=5,
                             scoring=make_scorer(mean_absolute_error))
    result[model_name] = scores
    print(model_name + ' is finished')

result = pd.DataFrame(result)
result.index = ['cv' + str(x) for x in range(1,6)]
result

model = LinearRegression().fit(train_X, train_y_ln)
print('intercept:' + str(model.intercept_))
sns.barplot(abs(model.coef_), continuous_feature_names)

观察线性回归模型,v6 v8 占比更大

model = LinearRegression().fit(train_X, train_y_ln)
print('intercept:' + str(model.intercept_))
sns.barplot(abs(model.coef_), continuous_feature_names)

加入了L1的线性回归模型–岭回归
我们发现power与userd_time特征非常重要。(L1正则化有助于生成一个稀疏权值矩阵，进而可以用于特征选择。)

model = Ridge().fit(train_X, train_y_ln)
print('intercept:' + str(model.intercept_))
sns.barplot(abs(model.coef_), continuous_feature_names)

加入了L2的先行回国模型–Lasso回归

model = Lasso().fit(train_X, train_y_ln)
print('intercept:' + str(model.intercept_))
sns.barplot(abs(model.coef_), continuous_feature_names)

4.4.3 -2 非线性模型

from sklearn.linear_model import LinearRegression
from sklearn.svm import SVC
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import RandomForestRegressor
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.neural_network import MLPRegressor
from xgboost.sklearn import XGBRegressor
from lightgbm.sklearn import LGBMRegressor

models = [LinearRegression(),
          DecisionTreeRegressor(),
          RandomForestRegressor(),
          GradientBoostingRegressor(),
          MLPRegressor(solver='lbfgs', max_iter=100),
          XGBRegressor(n_estimators = 100, objective='reg:squarederror'),
          LGBMRegressor(n_estimators = 100)]

result = dict()
for model in models:
    model_name =str(model).split('(')[0]
    scores = cross_val_score(model, X=train_X, y=train_y_ln, verbose=0, cv=5,  scoring=make_scorer(mean_absolute_error))
    result[model_name] = scores
    print(model_name + ' is finished')

result = pd.DataFrame(result)
result.index = ['cv' + str(x) for x in range(1,6)]
result

#4.4.4 模型调参 贪心算法、网络调参、贝叶斯调参

## LGB的参数集合

objective = ['regression', 'regression_l1', 'mape', 'huber', 'fair']

num_leaves = [3,5,10,15,20,40,55]
max_depth = [3,5,10,15,20,40,55]
bagging_fraction = []
feature_fraction = []
drop_rate = []

4.4.4 -1 贪心调参

	在对问题求解时，总是做出在当前看来是最好的选择。也就是说，不从整体最优上加以考虑，它所做出的仅仅是在某种意义上的局部最优解。

best_obj = dict()
for obj in objective:
    model = LGBMRegressor(objective=obj)
    score = np.mean(cross_val_score(model, X=train_X,y=train_y_ln, verbose=0, cv=5,
                                   scoring=make_scorer(mean_absolute_error)))
    best_obj[obj] = score
    
best_leaves = dict()
for leaves in num_leaves:
    model = LGBMRegressor(objective=min(best_obj.items(), key= lambda x:x[1])[0], num_leaves=leaves)
    score = np.mean(cross_val_score(model, X=train_X, y=train_y_ln, verbose=0, cv=5,
                                   scoring=make_scorer(mean_absolute_error)))
    best_leaves[leaves] = score

best_depth = dict()
for depth in max_depth:
    model = LGBMRegressor(objective=min(best_obj.items(), key=lambda x:x[1])[0],
                         num_leaves=min(best_leaves.items(), key=lambda x:x[1])[0],
                         max_depth=depth)
    score = np.mean(cross_val_score(model, X=train_X, y=train_y_ln, verbose=0, cv=5,
                                   scoring=make_scorer(mean_absolute_error)))
    best_depth[depth] = score

sns.lineplot(x=['0_initial','1_turning_obj','2_turning_leaves','3_turning_depth'],
             y=[0.143,min(best_obj.values()),min(best_leaves.values()),min(best_depth.values())])

##4.4.4 -2 Grid Search 调参 网格调参

	通过循环遍历，尝试每一种参数组合，返回最好的得分值的参数组合

from sklearn.model_selection import GridSearchCV

parameters = {'objective': objective, 'num_leaves': num_leaves, 'max_depth': max_depth}
model = LGBMRegressor()
clf = GridSearchCV(model, parameters, cv=5)
clf = clf.fit(train_X, train_y)
clf.best_params_
model = LGBMRegressor(objective='regression',
                     num_leaves=55,
                     max_depth=15)

np.mean(cross_val_score(model,X=train_X, y=train_y_ln, verbose=0, cv=5, scoring=make_scorer(mean_absolute_error)))

##4.4.4 -3 贝叶斯调参

	通过基于目标函数的过去评估结果建立替代函数（概率模型），来找到最小化目标函数的值。贝叶斯方法与随机或网格搜索的不同之处在于，它在尝试下一组超参数时，会参考之前的评估结果，因此可以省去很多无用功。

from bayes_opt import BayesianOptimization

def rf_cv(num_leaves, max_depth, subsample, min_child_samples):
    val = cross_val_score(
        LGBMRegressor(objective = 'regression_l1',
                     num_leaves=int(num_leaves),
                     max_depth=int(max_depth),
                     subsample = subsample,
                     min_child_samples = int(min_child_samples)
        ),
        X=train_X, y=train_y_ln, verbose=0, cv=5, scoring=make_scorer(mean_absolute_error)
    ).mean()
    return 1 - val
   
rf_bo = BayesianOptimization(
    rf_cv,
    {
        'num_leaves': (2,100),
        'max_depth': (2,100),
        'subsample': (0.1, 1),
        'min_child_samples' : (2,100)
    }
) 
rf_bo.maximize()
plt.figure(figsize(13,5))
sns.lineplot(x=['0_origin','1_log_transfer','2_L1_%_L2','3_change_model','4_parameter_turning'],
            y=[1.36,0.19,0.19,0.14,0.13])