0.基础知识学习
（1）线性回归模型：线性拟合，梯度下降调参，正态分布
（2）决策树模型：
西瓜书 第四章 决策树学习
（3）梯度提升树GBDT学习
CART树：二分树，通过寻找最优特征及其最佳切分点划分输入空间 + 剪枝操作
GBDT模型是集成模型，是很多CART树的线性相加
（4）XGboost模型
（5）LightGBM模型
import pandas as pd
import numpy as np
import warnings
warnings.filterwarnings('ignore')
# 调整数据类型，减少数据在内存中占用的空间
def reduce_mem_usage(df):
"""
遍历dataFrame中每一列数据并进行数据类型优化以减少内存使用
"""
start_mem = df.memory_usage().sum()
print('Memory usage of dataFrame is {:.2f}MB'.format(start_mem))
for col in df.columns:
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:
df[col] = df[col].astype(np.int8)
elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:
df[col] = df[col].astype(np.int16)
elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:
df[col] = df[col].astype(np.int32)
elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:
df[col] = df[col].astype(np.int64)
else:
if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:
df[col] = df[col].astype(np.float16)
elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:
df[col] = df[col].astype(np.float32)
else:
df[col] = df[col].astype(np.flaot64)
else:
df[col] = df[col].astype('category')
end_mem = df.memory_usage().sum()
print('Memory usage after optimization is：{:.2f}MB'.format(end_mem))
print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))
return df
sample_feature = reduce_mem_usage(pd.read_csv('data_for_tree.csv'))
Memory usage of dataFrame is 60507328.00MB
Memory usage after optimization is：15724107.00MB
Decreased by 74.0%
sample_feature.columns
Index(['name', 'model', 'brand', 'bodyType', 'fuelType', 'gearbox', 'power',
'kilometer', 'notRepairedDamage', 'seller', 'offerType', 'price', 'v_0',
'v_1', 'v_2', 'v_3', 'v_4', 'v_5', 'v_6', 'v_7', 'v_8', 'v_9', 'v_10',
'v_11', 'v_12', 'v_13', 'v_14', 'train', 'used_time', 'city',
'brand_amout', 'brand_price_average', 'brand_price_max',
'brand_price_median', 'brand_price_min', 'brand_price_std',
'brand_price_sum', 'power_bin'],
dtype='object')
sample_feature.head(10)
| name | model | brand | bodyType | fuelType | gearbox | power | kilometer | notRepairedDamage | seller | ... | used_time | city | brand_amout | brand_price_average | brand_price_max | brand_price_median | brand_price_min | brand_price_std | brand_price_sum | power_bin | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | 736 | 30.0 | 6 | 1.0 | 0.0 | 0.0 | 60 | 12.5 | 0.0 | 0 | ... | 4384.0 | 1.0 | 10192.0 | 3576.0 | 35990.0 | 1800.0 | 13.0 | 4564.0 | 36457520.0 | 5.0 |
| 1 | 2262 | 40.0 | 1 | 2.0 | 0.0 | 0.0 | 0 | 15.0 | - | 0 | ... | 4756.0 | 4.0 | 13656.0 | 9080.0 | 84000.0 | 6400.0 | 15.0 | 8992.0 | 124044600.0 | NaN |
| 2 | 14874 | 115.0 | 15 | 1.0 | 0.0 | 0.0 | 163 | 12.5 | 0.0 | 0 | ... | 4384.0 | 2.0 | 1458.0 | 9848.0 | 45000.0 | 8496.0 | 100.0 | 5424.0 | 14373814.0 | 16.0 |
| 3 | 71865 | 109.0 | 10 | 0.0 | 0.0 | 1.0 | 193 | 15.0 | 0.0 | 0 | ... | 7124.0 | NaN | 13992.0 | 8076.0 | 92900.0 | 5200.0 | 15.0 | 8248.0 | 113034208.0 | 19.0 |
| 4 | 111080 | 110.0 | 5 | 1.0 | 0.0 | 0.0 | 68 | 5.0 | 0.0 | 0 | ... | 1531.0 | 6.0 | 4664.0 | 3306.0 | 31500.0 | 2300.0 | 20.0 | 3344.0 | 15414322.0 | 6.0 |
| 5 | 137642 | 24.0 | 10 | 0.0 | 1.0 | 0.0 | 109 | 10.0 | 0.0 | 0 | ... | 2482.0 | 3.0 | 13992.0 | 8076.0 | 92900.0 | 5200.0 | 15.0 | 8248.0 | 113034208.0 | 10.0 |
| 6 | 2402 | 13.0 | 4 | 0.0 | 0.0 | 1.0 | 150 | 15.0 | 0.0 | 0 | ... | 6184.0 | 3.0 | 16576.0 | 8344.0 | 99999.0 | 6000.0 | 12.0 | 8088.0 | 138279072.0 | 14.0 |
| 7 | 165346 | 26.0 | 14 | 1.0 | 0.0 | 0.0 | 101 | 15.0 | 0.0 | 0 | ... | 6108.0 | 4.0 | 16072.0 | 3054.0 | 38990.0 | 1700.0 | 12.0 | 3606.0 | 49076652.0 | 10.0 |
| 8 | 2974 | 19.0 | 1 | 2.0 | 1.0 | 1.0 | 179 | 15.0 | 0.0 | 0 | ... | 4800.0 | 4.0 | 13656.0 | 9080.0 | 84000.0 | 6400.0 | 15.0 | 8992.0 | 124044600.0 | 17.0 |
| 9 | 82021 | 7.0 | 7 | 5.0 | 0.0 | 0.0 | 88 | 15.0 | 0.0 | 0 | ... | 6664.0 | NaN | 2360.0 | 4196.0 | 38900.0 | 2600.0 | 60.0 | 4752.0 | 9905909.0 | 8.0 |
10 rows × 38 columns
# 提取连续特征
continuous_feature_names = [x for x in sample_feature.columns if x not in ['price', 'brand', 'model', 'brand']]
# 处理数据中的缺失值（删除），将‘-’替换为0并将‘notRepairedDamage’数据类型转换为float32
sample_feature = sample_feature.dropna().replace('-', 0).reset_index(drop=True)
sample_feature['notRepairedDamage'] = sample_feature['notRepairedDamage'].astype(np.float32)
train = sample_feature[continuous_feature_names + ['price']]
train_X = train[continuous_feature_names]
train_y = train['price']
1.1 简单建模
from sklearn.linear_model import LinearRegression
model = LinearRegression(normalize=True)
model = model.fit(train_X, train_y)
# 查看训练的线性回归模型的截距与权重
'intercept:' + str(model.intercept_)
sorted(dict(zip(continuous_feature_names, model.coef_)).items(), key=lambda x:x[1], reverse=True)
[('v_6', 3367077.4406224387),
('v_8', 700656.3543472802),
('v_9', 170626.24091385124),
('v_7', 32318.173103516365),
('v_12', 20480.56267793342),
('v_3', 17871.47564683771),
('v_11', 11482.123138800021),
('v_13', 11263.399851895827),
('v_10', 2681.353980785434),
('gearbox', 881.832820328575),
('fuelType', 363.90247374080616),
('bodyType', 189.58552716184977),
('city', 44.953622398283215),
('power', 28.557627369989373),
('brand_price_median', 0.5103099160112271),
('brand_price_std', 0.4503275546866113),
('brand_amout', 0.14881138893606274),
('brand_price_max', 0.0031902053613149382),
('seller', 2.9080547392368317e-07),
('train', 6.891787052154541e-08),
('offerType', -4.839152097702026e-06),
('brand_price_sum', -2.175000814125744e-05),
('name', -0.0002981582332371014),
('used_time', -0.0025261487756046805),
('brand_price_average', -0.4048195975444779),
('brand_price_min', -2.2467183600593943),
('power_bin', -34.45676039711011),
('v_14', -274.91399236959336),
('kilometer', -372.89762118323057),
('notRepairedDamage', -495.2282384086448),
('v_0', -2044.689562386807),
('v_5', -11046.342844467923),
('v_4', -15123.010532417948),
('v_2', -26106.90644371924),
('v_1', -45560.92511426905)]
from matplotlib import pyplot as plt
subsample_index = np.random.randint(low=0, high=len(train_y), size=50)
# 绘制特征v9的值与标签的散点图
plt.scatter(train_X['v_9'][subsample_index], train_y[subsample_index], color='black')
plt.scatter(train_X['v_9'][subsample_index], model.predict(train_X.loc[subsample_index]), color='blue')
plt.xlabel('v_9')
plt.ylabel('price')
plt.legend(['True Price', 'Predicted Price'], loc='upper right')
print('The predicted price is obvious different from true price')
plt.show()
通过上图可以发现模型的预测结果（蓝色点）与真实结果（黑色点）的分布差异较大，且部分预测值出现了小于0的情况，
说明模型存在一定的问题
import seaborn as sns
print('It is clear to see the price shows a typical exponential distribution')
plt.figure(figsize=(15,5))
plt.subplot(1,2,1)
sns.distplot(train_y)
plt.subplot(1,2,2)
sns.distplot(train_y[train_y < np.quantile(train_y, 0.9)])
It is clear to see the price shows a typical exponential distribution
# 对标签进行log（x + 1）变换，使其贴近于正态分布
train_y_ln = np.log(train_y + 1)
print('The transformed price seems like normal distribution')
plt.figure(figsize=(15,5))
plt.subplot(1,2,1)
sns.distplot(train_y_ln)
plt.subplot(1,2,2)
sns.distplot(train_y_ln[train_y_ln < np.quantile(train_y_ln, 0.9)])
The transformed price seems like normal distribution
model = model.fit(train_X, train_y_ln)
print('intercept:' + str(model.intercept_))
sorted(dict(zip(continuous_feature_names, model.coef_)).items(), key=lambda x:x[1], reverse=True)
intercept:18.74880341534744
[('v_9', 8.050812690369153),
('v_5', 5.754994162620334),
('v_12', 1.62093530763775),
('v_1', 1.4779570497262025),
('v_11', 1.169744490862118),
('v_13', 0.9411182304172357),
('v_7', 0.7119510355048398),
('v_3', 0.6851314488405563),
('v_0', 0.008645108332593266),
('power_bin', 0.008483677741707147),
('gearbox', 0.007926459579133759),
('fuelType', 0.006684066538356764),
('bodyType', 0.004516720629709567),
('power', 0.0007176637371205766),
('brand_price_min', 3.3366062098929906e-05),
('brand_amout', 2.8979529047958453e-06),
('brand_price_median', 1.2322281792545508e-06),
('brand_price_std', 6.517052329781775e-07),
('brand_price_average', 6.336678789883344e-07),
('brand_price_max', 6.191737965170084e-07),
('seller', 2.319779923709575e-10),
('offerType', 1.141984284913633e-10),
('train', -1.5916157281026244e-12),
('brand_price_sum', -1.512410856518073e-10),
('name', -7.021723724238846e-08),
('used_time', -4.126537154502869e-06),
('city', -0.0022172506125169183),
('v_14', -0.004285615931568405),
('kilometer', -0.01383590363034406),
('notRepairedDamage', -0.2702944026708757),
('v_4', -0.8320763999815677),
('v_2', -0.950489908625956),
('v_10', -1.6271621034357788),
('v_8', -40.35060772204042),
('v_6', -238.78518046195867)]
# 再次进行可视化以观察效果
plt.scatter(train_X['v_9'][subsample_index], train_y[subsample_index], color='black')
plt.scatter(train_X['v_9'][subsample_index], np.exp(model.predict(train_X.loc[subsample_index]))-1, color='blue')
plt.xlabel('v_9')
plt.ylabel('price')
plt.legend(['True Price', 'Predicted Price'], loc='upper right')
plt.show()
1.2 五折交叉验证
from sklearn.model_selection import cross_val_score
from sklearn.metrics import mean_absolute_error, make_scorer
def log_transfer(func):
def wrapper(y, yhat):
result = func(np.log(y), np.nan_to_num(np.log(yhat)))
return result
return wrapper
scores = cross_val_score(model, X=train_X, y=train_y, verbose=1, cv=5, scoring=make_scorer(log_transfer(mean_absolute_error)))
[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.
[Parallel(n_jobs=1)]: Done 5 out of 5 | elapsed: 2.0s finished
# 使用线性回归模型，对未处理标签的特征数据进行五折交叉验证(Error 1.36)
print('AVG:', np.mean(scores))
AVG: 1.365429593439596
# 使用线性回归模型，对处理过标签的特征数据进行五折交叉验证（Error 0.19）
scores = cross_val_score(model, X=train_X, y=train_y_ln, verbose=1, cv=5, scoring=make_scorer(mean_absolute_error))
[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.
[Parallel(n_jobs=1)]: Done 5 out of 5 | elapsed: 1.4s finished
print('AVG:', np.mean(scores))
AVG: 0.19323301794380213
scores = pd.DataFrame(scores.reshape(1, -1))
scores.columns = ['cv' + str(x) for x in range(1, 6)]
scores
| cv1 | cv2 | cv3 | cv4 | cv5 | |
|---|---|---|---|---|---|
| 0 | 0.1908 | 0.193762 | 0.194131 | 0.191823 | 0.19565 |
2.3 模拟真实业务情况
但在事实上，由于我们并不具有预知未来的能力，五折交叉验证在某些与时间相关的数据集上反而反映了不真实的情况。
通过2018年的二手车价格预测2017年的二手车价格，这显然是不合理的，因此我们还可以采用时间顺序对数据集进行分隔。
在本例中，我们选用靠前时间的4/5样本当作训练集，靠后时间的1/5当作验证集，最终结果与五折交叉验证差距不大
import datetime
sample_feature = sample_feature.reset_index(drop=True)
split_point = len(sample_feature) // 5 * 4
train = sample_feature.loc[:split_point].dropna()
val = sample_feature.loc[split_point:].dropna()
train_X = train[continuous_feature_names]
train_y_ln = np.log(train['price'] + 1)
val_X = val[continuous_feature_names]
val_y_ln = np.log(val['price'] + 1)
model = model.fit(train_X, train_y_ln)
mean_absolute_error(val_y_ln, model.predict(val_X))
0.19566623218534546
2.4 绘制学习率曲线和验证曲线
from sklearn.model_selection import learning_curve, validation_curve
? learning_curve
def plot_learning_curve(estimator , title, X, y, ylim=None, cv=None, n_jobs=1, train_size=np.linspace(.1, 1.0, 5)):
plt.figure()
plt.title(title)
if ylim is not None:
plt.ylim(*ylim)
plt.xlabel('Training example')
plt.ylabel('score')
train_sizes, train_scores, test_scores = learning_curve(estimator, X, y, cv=cv, n_jobs=n_jobs, train_sizes=train_size, scoring = make_scorer(mean_absolute_error))
train_scores_mean = np.mean(train_scores, axis=1)
train_scores_std = np.std(train_scores, axis=1)
test_scores_mean = np.mean(test_scores, axis=1)
test_scores_std = np.std(test_scores, axis=1)
plt.grid()#区域
plt.fill_between(train_sizes, train_scores_mean - train_scores_std,
train_scores_mean + train_scores_std, alpha=0.1,
color="r")
plt.fill_between(train_sizes, test_scores_mean - test_scores_std,
test_scores_mean + test_scores_std, alpha=0.1,
color="g")
plt.plot(train_sizes, train_scores_mean, 'o-', color='r',
label="Training score")
plt.plot(train_sizes, test_scores_mean,'o-',color="g",
label="Cross-validation score")
plt.legend(loc="best")
return plt
plot_learning_curve(LinearRegression(), 'Liner_model', train_X[:1000], train_y_ln[:1000], ylim=(0.0, 0.5), cv=5, n_jobs=1)
- 多种模型对比
3.1 线性模型&嵌入式特征选择
在过滤式和包裹式特征选择方法中，特征选择过程与学习器训练过程有明显的分别。而嵌入式特征选择在学习器训练过程中自动地进行特征选择。嵌入式选择最常用的是L1正则化与L2正则化。在对线性回归模型加入两种正则化方法后，他们分别变成了岭回归与Lasso回归。
from sklearn.linear_model import LinearRegression
from sklearn.linear_model import Ridge
from sklearn.linear_model import Lasso
models = [LinearRegression(),
Ridge(),
Lasso()]
result = dict()
for model in models:
model_name = str(model).split('(')[0]
scores = cross_val_score(model, X=train_X, y=train_y_ln, verbose=0, cv=5, scoring=make_scorer((mean_absolute_error)))
result[model_name] = scores
print(model_name + 'is finished')
LinearRegressionis finished
Ridgeis finished
Lassois finished
# 对三种方法的效果对比
result = pd.DataFrame(result)
result.index = ['cv' + str(x) for x in range(1, 6)]
result
| LinearRegression | Ridge | Lasso | |
|---|---|---|---|
| cv1 | 0.190303 | 0.194213 | 0.383467 |
| cv2 | 0.193318 | 0.197295 | 0.383593 |
| cv3 | 0.192839 | 0.196655 | 0.383413 |
| cv4 | 0.193262 | 0.197033 | 0.382736 |
| cv5 | 0.191895 | 0.195746 | 0.379170 |
model = LinearRegression().fit(train_X, train_y_ln)
print('intercept:' + str(model.intercept_))
sns.barplot(abs(model.coef_), continuous_feature_names)
intercept:17.249525951718784
L2正则化在拟合过程中通常都倾向于让权值尽可能小，最后构造一个所有参数都比较小的模型。因为一般认为参数值小的模型比较简单，能适应不同的数据集，也在一定程度上避免了过拟合现象。可以设想一下对于一个线性回归方程，若参数很大，那么只要数据偏移一点点，就会对结果造成很大的影响；但如果参数足够小，数据偏移得多一点也不会对结果造成什么影响，专业一点的说法是『抗扰动能力强』
model = Ridge().fit(train_X, train_y_ln)
print('intercept:' + str(model.intercept_))
sns.barplot(abs(model.coef_), continuous_feature_names)
intercept:3.118190236616388
model = Lasso().fit(train_X, train_y_ln)
print('intercept:' + str(model.intercept_))
sns.barplot(abs(model.coef_), continuous_feature_names)
intercept:8.669914057713994
3.2 非线性模型
from sklearn.linear_model import LinearRegression
from sklearn.svm import SVC
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import RandomForestRegressor
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.neural_network import MLPRegressor
from xgboost.sklearn import XGBRegressor
from lightgbm.sklearn import LGBMRegressor
9820
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
