特征工程与基线模型

特征工程

概述：

数据和特征决定了机器学习的上限，而模型和算法只是逼近这个上限。

特征工程本质是一项工程活动，目的是最大限度地从原始数据中提取特征以供算法和模型使用。

要注重思考自己构造的特征是否合理、特征是否足够表达要解决的问题。

方法：

1. 基于数据类型
2. 基于多分析视角

特征选择：

1. 过滤法（Filter）
2. 包装法（Wrapper）
3. 嵌入法（Embedded）

构造线下验证集

构造线下验证集需要考虑三个方面的问题

1. 评估穿越
2. 训练集和测试集的特征差异性
3. 训练集和测试集的分布差异性

基线模型

# -*- coding: utf-8 -*-
'''
@Time    : 2021/1/29 18:21
@Author  : Hjh
@File    : baseline.py
'''

# 导入相关应用包
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt

import lightgbm as lgb
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import OneHotEncoder

# 忽略警告信息
import warnings

warnings.filterwarnings("ignore")

# 调用matplotlib.pyplot的绘图函数plot()进行绘图的时候，
# 或者生成一个figure画布的时候，可以直接在你的python console里面生成图像。
# %matplotlib inline

# 读取数据
path = './dataset/'
train = pd.read_csv(path + 'train.csv')  # 训练集
test = pd.read_csv(path + 'test.csv')  # 测试集

# 特征工程
# 利用count()函数和nunique()函数生成特征：反映样本调用api,tid,index的频率信息
def simple_sts_features(df):
    simple_fea = pd.DataFrame()
    simple_fea['file_id'] = df['file_id'].unique()
    simple_fea = simple_fea.sort_values('file_id')

    df_grp = df.groupby('file_id')
    simple_fea['file_id_api_count'] = df_grp['api'].count().values
    simple_fea['file_id_api_nunique'] = df_grp['api'].nunique().values

    simple_fea['file_id_tid_count'] = df_grp['tid'].count().values
    simple_fea['file_id_tid_nunique'] = df_grp['tid'].nunique().values

    simple_fea['file_id_index_count'] = df_grp['index'].count().values
    simple_fea['file_id_index_nunique'] = df_grp['index'].nunique().values

    return simple_fea

# 利用mean()函数、min()函数、std()函数、max()函数生成特征：tid,index可认为是数值特征，可提取对应的统计特征
def simple_numerical_sts_features(df):
    simple_numerical_fea = pd.DataFrame()
    simple_numerical_fea['file_id'] = df['file_id'].unique()
    simple_numerical_fea = simple_numerical_fea.sort_values('file_id')

    df_grp = df.groupby('file_id')

    simple_numerical_fea['file_id_tid_mean'] = df_grp['tid'].mean().values
    simple_numerical_fea['file_id_tid_min'] = df_grp['tid'].min().values
    simple_numerical_fea['file_id_tid_std'] = df_grp['tid'].std().values
    simple_numerical_fea['file_id_tid_max'] = df_grp['tid'].max().values

    simple_numerical_fea['file_id_index_mean'] = df_grp['index'].mean().values
    simple_numerical_fea['file_id_index_min'] = df_grp['index'].min().values
    simple_numerical_fea['file_id_index_std'] = df_grp['index'].std().values
    simple_numerical_fea['file_id_index_max'] = df_grp['index'].max().values

    return simple_numerical_fea

#利用定义的特征生成函数，并生成训练集和测试集的统计特征
#（1）反映样本调用api,tid,index的频率信息和统计特征
simple_train_fea1 = simple_sts_features(train)
simple_test_fea1 = simple_sts_features(test)
#（2）反映tid、index等数值特征的统计特征
simple_train_fea2 = simple_numerical_sts_features(train)
simple_test_fea2 = simple_numerical_sts_features(test)

# 基线构建
# 获取标签
train_label = train[['file_id', 'label']].drop_duplicates(subset=['file_id', 'label'], keep='first')
test_submit = test[['file_id']].drop_duplicates(subset=['file_id'], keep='first')
# 训练集&测试集构建
train_data = train_label.merge(simple_train_fea1, on='file_id', how='left')
train_data = train_data.merge(simple_train_fea2, on='file_id', how='left')

test_submit = test_submit.merge(simple_test_fea1, on='file_id', how='left')
test_submit = test_submit.merge(simple_test_fea2, on='file_id', how='left')


# 模型评估函数
def lgb_logloss(preds, data):
    labels_ = data.get_label()
    classes_ = np.unique(labels_)
    preds_prob = []
    for i in range(len(classes_)):
        preds_prob.append(preds[i * len(labels_):(i + 1) * len(labels_)])

    preds_prob_ = np.vstack(preds_prob)

    loss = []
    for i in range(preds_prob_.shape[1]):  # 样本个数
        sum_ = 0
        for j in range(preds_prob_.shape[0]):  # 类别个数
            pred = preds_prob_[j, i]  # 第i个样本预测为第j类的概率
            if j == labels_[i]:
                sum_ += np.log(pred)
            else:
                sum_ += np.log(1 - pred)
        loss.append(sum_)
    return 'loss is: ', -1 * (np.sum(loss) / preds_prob_.shape[1]), False


# 模型验证。因为数据与时间的相关性不是非常大，所以使用传统的5折交叉验证来构建线下验证集
train_features = [col for col in train_data.columns if col not in ['label', 'file_id']]
train_label = 'label'

#%%time
import time
from sklearn.model_selection import StratifiedKFold, KFold

start1 = time.perf_counter()
end1 = time.perf_counter()

params = {
    'task': 'train',
    'num_leaves': 255,
    'objective': 'multiclass',
    'num_class': 8,
    'min_data_in_leaf': 50,
    'learning_rate': 0.05,
    'feature_fraction': 0.85,
    'bagging_fraction': 0.85,
    'bagging_freq': 5,
    'max_bin': 128,
    'random_state': 100
}

folds = KFold(n_splits=5, shuffle=True, random_state=15)
oof = np.zeros(len(train))

predict_res = 0
models = []
for fold_, (trn_idx, val_idx) in enumerate(folds.split(train_data)):
    print("fold n°{}".format(fold_))
    trn_data = lgb.Dataset(train_data.iloc[trn_idx][train_features], label=train_data.iloc[trn_idx][train_label].values)
    val_data = lgb.Dataset(train_data.iloc[val_idx][train_features], label=train_data.iloc[val_idx][train_label].values)

    clf = lgb.train(params, trn_data, num_boost_round=2000, valid_sets=[trn_data, val_data], verbose_eval=50,
                    early_stopping_rounds=100, feval=lgb_logloss)
    models.append(clf)

print("final is in : %s Seconds " % (end1-start1))

# 特征重要性分析
feature_importance = pd.DataFrame()
feature_importance['fea_name'] = train_features
feature_importance['fea_imp'] = clf.feature_importance()
feature_importance = feature_importance.sort_values('fea_imp',ascending = False)

#模型测试
pred_res = 0
fold = 5
for model in models:
    pred_res +=model.predict(test_submit[train_features]) * 1.0 / fold
test_submit['prob0'] = 0
test_submit['prob1'] = 0
test_submit['prob2'] = 0
test_submit['prob3'] = 0
test_submit['prob4'] = 0
test_submit['prob5'] = 0
test_submit['prob6'] = 0
test_submit['prob7'] = 0
test_submit[['prob0','prob1','prob2','prob3','prob4','prob5','prob6','prob7']] = pred_res
test_submit[['file_id','prob0','prob1','prob2','prob3','prob4','prob5','prob6','prob7']].to_csv('baseline.csv',index = None)