特征工程
概述:
数据和特征决定了机器学习的上限,而模型和算法只是逼近这个上限。
特征工程本质是一项工程活动,目的是最大限度地从原始数据中提取特征以供算法和模型使用。
要注重思考自己构造的特征是否合理、特征是否足够表达要解决的问题。
方法:
- 基于数据类型
- 基于多分析视角
特征选择:
- 过滤法(Filter)
- 包装法(Wrapper)
- 嵌入法(Embedded)
构造线下验证集
构造线下验证集需要考虑三个方面的问题
- 评估穿越
- 训练集和测试集的特征差异性
- 训练集和测试集的分布差异性
基线模型
# -*- coding: utf-8 -*-
'''
@Time : 2021/1/29 18:21
@Author : Hjh
@File : baseline.py
'''
# 导入相关应用包
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import lightgbm as lgb
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import OneHotEncoder
# 忽略警告信息
import warnings
warnings.filterwarnings("ignore")
# 调用matplotlib.pyplot的绘图函数plot()进行绘图的时候,
# 或者生成一个figure画布的时候,可以直接在你的python console里面生成图像。
# %matplotlib inline
# 读取数据
path = './dataset/'
train = pd.read_csv(path + 'train.csv') # 训练集
test = pd.read_csv(path + 'test.csv') # 测试集
# 特征工程
# 利用count()函数和nunique()函数生成特征:反映样本调用api,tid,index的频率信息
def simple_sts_features(df):
simple_fea = pd.DataFrame()
simple_fea['file_id'] = df['file_id'].unique()
simple_fea = simple_fea.sort_values('file_id')
df_grp = df.groupby('file_id')
simple_fea['file_id_api_count'] = df_grp['api'].count().values
simple_fea['file_id_api_nunique'] = df_grp['api'].nunique().values
simple_fea['file_id_tid_count'] = df_grp['tid'].count().values
simple_fea['file_id_tid_nunique'] = df_grp['tid'].nunique().values
simple_fea['file_id_index_count'] = df_grp['index'].count().values
simple_fea['file_id_index_nunique'] = df_grp['index'].nunique().values
return simple_fea
# 利用mean()函数、min()函数、std()函数、max()函数生成特征:tid,index可认为是数值特征,可提取对应的统计特征
def simple_numerical_sts_features(df):
simple_numerical_fea = pd.DataFrame()
simple_numerical_fea['file_id'] = df['file_id'].unique()
simple_numerical_fea = simple_numerical_fea.sort_values('file_id')
df_grp = df.groupby('file_id')
simple_numerical_fea['file_id_tid_mean'] = df_grp['tid'].mean().values
simple_numerical_fea['file_id_tid_min'] = df_grp['tid'].min().values
simple_numerical_fea['file_id_tid_std'] = df_grp['tid'].std().values
simple_numerical_fea['file_id_tid_max'] = df_grp['tid'].max().values
simple_numerical_fea['file_id_index_mean'] = df_grp['index'].mean().values
simple_numerical_fea['file_id_index_min'] = df_grp['index'].min().values
simple_numerical_fea['file_id_index_std'] = df_grp['index'].std().values
simple_numerical_fea['file_id_index_max'] = df_grp['index'].max().values
return simple_numerical_fea
#利用定义的特征生成函数,并生成训练集和测试集的统计特征
#(1)反映样本调用api,tid,index的频率信息和统计特征
simple_train_fea1 = simple_sts_features(train)
simple_test_fea1 = simple_sts_features(test)
#(2)反映tid、index等数值特征的统计特征
simple_train_fea2 = simple_numerical_sts_features(train)
simple_test_fea2 = simple_numerical_sts_features(test)
# 基线构建
# 获取标签
train_label = train[['file_id', 'label']].drop_duplicates(subset=['file_id', 'label'], keep='first')
test_submit = test[['file_id']].drop_duplicates(subset=['file_id'], keep='first')
# 训练集&测试集构建
train_data = train_label.merge(simple_train_fea1, on='file_id', how='left')
train_data = train_data.merge(simple_train_fea2, on='file_id', how='left')
test_submit = test_submit.merge(simple_test_fea1, on='file_id', how='left')
test_submit = test_submit.merge(simple_test_fea2, on='file_id', how='left')
# 模型评估函数
def lgb_logloss(preds, data):
labels_ = data.get_label()
classes_ = np.unique(labels_)
preds_prob = []
for i in range(len(classes_)):
preds_prob.append(preds[i * len(labels_):(i + 1) * len(labels_)])
preds_prob_ = np.vstack(preds_prob)
loss = []
for i in range(preds_prob_.shape[1]): # 样本个数
sum_ = 0
for j in range(preds_prob_.shape[0]): # 类别个数
pred = preds_prob_[j, i] # 第i个样本预测为第j类的概率
if j == labels_[i]:
sum_ += np.log(pred)
else:
sum_ += np.log(1 - pred)
loss.append(sum_)
return 'loss is: ', -1 * (np.sum(loss) / preds_prob_.shape[1]), False
# 模型验证。因为数据与时间的相关性不是非常大,所以使用传统的5折交叉验证来构建线下验证集
train_features = [col for col in train_data.columns if col not in ['label', 'file_id']]
train_label = 'label'
#%%time
import time
from sklearn.model_selection import StratifiedKFold, KFold
start1 = time.perf_counter()
end1 = time.perf_counter()
params = {
'task': 'train',
'num_leaves': 255,
'objective': 'multiclass',
'num_class': 8,
'min_data_in_leaf': 50,
'learning_rate': 0.05,
'feature_fraction': 0.85,
'bagging_fraction': 0.85,
'bagging_freq': 5,
'max_bin': 128,
'random_state': 100
}
folds = KFold(n_splits=5, shuffle=True, random_state=15)
oof = np.zeros(len(train))
predict_res = 0
models = []
for fold_, (trn_idx, val_idx) in enumerate(folds.split(train_data)):
print("fold n°{}".format(fold_))
trn_data = lgb.Dataset(train_data.iloc[trn_idx][train_features], label=train_data.iloc[trn_idx][train_label].values)
val_data = lgb.Dataset(train_data.iloc[val_idx][train_features], label=train_data.iloc[val_idx][train_label].values)
clf = lgb.train(params, trn_data, num_boost_round=2000, valid_sets=[trn_data, val_data], verbose_eval=50,
early_stopping_rounds=100, feval=lgb_logloss)
models.append(clf)
print("final is in : %s Seconds " % (end1-start1))
# 特征重要性分析
feature_importance = pd.DataFrame()
feature_importance['fea_name'] = train_features
feature_importance['fea_imp'] = clf.feature_importance()
feature_importance = feature_importance.sort_values('fea_imp',ascending = False)
#模型测试
pred_res = 0
fold = 5
for model in models:
pred_res +=model.predict(test_submit[train_features]) * 1.0 / fold
test_submit['prob0'] = 0
test_submit['prob1'] = 0
test_submit['prob2'] = 0
test_submit['prob3'] = 0
test_submit['prob4'] = 0
test_submit['prob5'] = 0
test_submit['prob6'] = 0
test_submit['prob7'] = 0
test_submit[['prob0','prob1','prob2','prob3','prob4','prob5','prob6','prob7']] = pred_res
test_submit[['file_id','prob0','prob1','prob2','prob3','prob4','prob5','prob6','prob7']].to_csv('baseline.csv',index = None)