CCF 乘用车细分市场销量预测 35个特征lgb单模0.6253

CCF 乘用车细分市场销量预测 35个特征lgb单模0.6253

import sys
# import shap
import numpy as np
import pandas as pd
import os 
import gc
from tqdm import tqdm, tqdm_notebook 
from sklearn.model_selection import StratifiedKFold, KFold
from sklearn.metrics import f1_score, roc_auc_score
from sklearn.metrics import mean_squared_error as mse
from sklearn.preprocessing import LabelEncoder
import datetime
import time
import matplotlib.pyplot as plt
import lightgbm as lgb
from sklearn.cluster import KMeans
import xgboost as xgb
from sklearn.externals import joblib
import warnings
warnings.simplefilter(action='ignore', category=FutureWarning)
warnings.filterwarnings('ignore')
train_sales  = pd.read_csv('../input/train_sales_data.csv')
train_search = pd.read_csv('../input/train_search_data.csv')
train_user   = pd.read_csv('../input/train_user_reply_data.csv')
evaluation_public = pd.read_csv('../input/evaluation_public.csv')
submit_example    = pd.read_csv('../input/submit_example.csv')
data = pd.concat([train_sales, evaluation_public], ignore_index=True)
data = data.merge(train_search, 'left', on=['province', 'adcode', 'model', 'regYear', 'regMonth'])
data = data.merge(train_user, 'left', on=['model', 'regYear', 'regMonth'])
# data=pd.concat([data, k_mean_data], axis=1)
data['label'] = data['salesVolume']
data['id'] = data['id'].fillna(0).astype(int)
data['bodyType'] = data['model'].map(train_sales.drop_duplicates('model').set_index('model')['bodyType'])
#LabelEncoder
for i in ['bodyType', 'model']:
    data[i] = data[i].map(dict(zip(data[i].unique(), range(data[i].nunique()))))
data['mt'] = (data['regYear'] - 2016) * 12 + data['regMonth']

def get_stat_feature(df_,): 
    df = df_.copy()
    stat_feat = []
    stat_feat_2=[]
    stat_feat_3 = []
    stat_feat_4 = []
    df['model_adcode'] = df['adcode'] + df['model']
    df['model_adcode_mt'] = df['model_adcode'] * 100 + df['mt']
    for col in ['label']:
        # 历史销量数据特征
        for i in [1,2,3,4,5,6,8,9,10,11,12,13,14,15,16]:
            stat_feat.append('shift_model_adcode_mt_{}_{}'.format(col,i))
            stat_feat_2.append('shift_model_adcode_mt_{}_{}'.format(col,i))
            df['model_adcode_mt_{}_{}'.format(col,i)] = df['model_adcode_mt'] + i#新加一列值，等于车型*省*时间+i，寻求i个月前的值，将model_adcode_mt_作为索引
            df_last = df[~df[col].isnull()].set_index('model_adcode_mt_{}_{}'.format(col,i))
            df['shift_model_adcode_mt_{}_{}'.format(col,i)] = df['model_adcode_mt'].map(df_last[col])#后者索引是31000002开始，前者少i，取前面的匹配后面索引成功，就取值
    for col in ['popularity']:
        # 历史销量数据特征
        for i in [1,2,3,10,11,12]:#popularity只取一部分
            stat_feat.append('shift_model_adcode_mt_{}_{}'.format(col,i))
            stat_feat_2.append('shift_model_adcode_mt_{}_{}'.format(col,i))
            df['model_adcode_mt_{}_{}'.format(col,i)] = df['model_adcode_mt'] + i#新加一列值，等于车型*省*时间+i，寻求i个月前的值，将model_adcode_mt_作为索引
            df_last = df[~df[col].isnull()].set_index('model_adcode_mt_{}_{}'.format(col,i))
            df['shift_model_adcode_mt_{}_{}'.format(col,i)] = df['model_adcode_mt'].map(df_last[col])#后者索引是31000002开始，前者少i，取前面的匹配后面索引成功，就取值
    df["increase16_4"]=(df["shift_model_adcode_mt_label_16"]-df["shift_model_adcode_mt_label_4"])/df["shift_model_adcode_mt_label_16"]#同比一年前的增长
    mean=pd.DataFrame(df.groupby(["model","mt"]).shift_model_adcode_mt_label_12.agg({"mean_province":"mean",
                                                                          "min_province":"min",}))
    df=pd.merge(df,mean,on=["model","mt"],how="left")
    mean=pd.DataFrame(df.groupby(["model","mt"]).shift_model_adcode_mt_label_15.agg({"mean_province_15":"mean",}))
    df=pd.merge(df,mean,on=["model","mt"],how="left")
    mean=pd.DataFrame(df.groupby(["model","mt"]).shift_model_adcode_mt_label_3.agg({"mean_province_3":"mean",}))
    df=pd.merge(df,mean,on=["model","mt"],how="left")
    mean=pd.DataFrame(df.groupby(["model","mt"]).shift_model_adcode_mt_label_16.agg({"mean_province_16":"mean",}))
    df=pd.merge(df,mean,on=["model","mt"],how="left")
    mean=pd.DataFrame(df.groupby(["model","mt"]).shift_model_adcode_mt_label_4.agg({"mean_province_4":"mean",}))
    df=pd.merge(df,mean,on=["model","mt"],how="left")
	#另一种统计方式
    mean=pd.DataFrame(df.groupby(["adcode","mt"]).shift_model_adcode_mt_label_15.agg({"mean_Month_15":"mean"}))
    df=pd.merge(df,mean,on=["adcode","mt"],how="left") 
    mean=pd.DataFrame(df.groupby(["adcode","mt"]).shift_model_adcode_mt_label_3.agg({"mean_Month_3":"mean"}))
    df=pd.merge(df,mean,on=["adcode","mt"],how="left") 
    mean=pd.DataFrame(df.groupby(["adcode","mt"]).shift_model_adcode_mt_label_16.agg({"mean_Month_16":"mean"}))
    df=pd.merge(df,mean,on=["adcode","mt"],how="left") 
    mean=pd.DataFrame(df.groupby(["adcode","mt"]).shift_model_adcode_mt_label_4.agg({"mean_Month_4":"mean"}))
    df=pd.merge(df,mean,on=["adcode","mt"],how="left") 
    #基于统计特征的increase，强特
    df["increase_mean_province_16_4"]=(df["mean_province_16"]-df["mean_province_4"])/df["mean_province_16"]
    df["increase_mean_province_15_3"]=(df["mean_province_15"]-df["mean_province_3"])/df["mean_province_15"]
    df["increase_mean_Month_15_3"]=(df["mean_Month_15"]-df["mean_Month_3"])/df["mean_Month_15"]
    df["increase_mean_Month_16_4"]=(df["mean_Month_16"]-df["mean_Month_4"])/df["mean_Month_16"]
    mean=pd.DataFrame(df.groupby(["adcode","mt"]).shift_model_adcode_mt_label_12.agg({"mean_Month":"mean",}))
    df=pd.merge(df,mean,on=["adcode","mt"],how="left")
 	#几个月sum
    df["sum_1"]=df["shift_model_adcode_mt_label_11"].values+df["shift_model_adcode_mt_label_12"].values+df["shift_model_adcode_mt_label_1"].values+df["shift_model_adcode_mt_label_2"].values
    df["sum_2"]=df["shift_model_adcode_mt_label_12"].values+df["shift_model_adcode_mt_label_1"].values
    df["sum_3"]=df["shift_model_adcode_mt_label_3"].values+df["shift_model_adcode_mt_label_2"].values+df["shift_model_adcode_mt_label_1"].values
    stat_feat_4 = ["mean_province","min_province","mean_Month","sum_1","sum_2","sum_3","increase16_4",
                   "increase_mean_province_15_3","increase_mean_Month_15_3","increase_mean_province_16_4","increase_mean_Month_16_4"]#所有统计特征
    stat_feat.remove("shift_model_adcode_mt_label_15")#删掉两个特征
    stat_feat.remove("shift_model_adcode_mt_label_16")
    return df,stat_feat+stat_feat_3+stat_feat_4
	#下面基本和鱼佬一样
def score(data, pred='pred_label', label='label', group='model'):
    data['pred_label'] = data['pred_label'].apply(lambda x: 0 if x < 0 else x).round().astype(int)
    data_agg = data.groupby('model').agg({
        pred:  list,
        label: [list, 'mean']
    }).reset_index()
    data_agg.columns = ['_'.join(col).strip() for col in data_agg.columns]
    nrmse_score = []
    for raw in data_agg[['{0}_list'.format(pred), '{0}_list'. format(label), '{0}_mean'.format(label)]].values:
        nrmse_score.append(mse(raw[0], raw[1]) ** 0.5 / raw[2] )
    print(1 - np.mean(nrmse_score))
    return 1 - np.mean(nrmse_score)
def get_model_type(train_x,train_y,valid_x,valid_y,m_type='lgb',i=0):   
    if m_type == 'lgb':
        model = lgb.LGBMRegressor(
                                num_leaves=2**5-1, reg_alpha=0.25, reg_lambda=0.25, objective='mse',
                                max_depth=-1, learning_rate=0.05, min_child_samples=10, random_state=2019,
                                n_estimators=2000, subsample=0.9, colsample_bytree=0.7,num_threads= -1,
                                )
        model.fit(train_x, train_y, 
              eval_set=[(train_x, train_y),(valid_x, valid_y)], 
              categorical_feature=cate_feat, 
              early_stopping_rounds=100, verbose=100)
        joblib.dump(model, "lgbm_"+str(i)+".m")
        print("lgb_model_%d has saved"%i)
    elif m_type == 'xgb':
        model = xgb.XGBRegressor(
                                max_depth=5 , learning_rate=0.05, n_estimators=2000, 
                                objective='reg:gamma', tree_method = 'hist',subsample=0.9, 
                                colsample_bytree=0.7, min_child_samples=5,eval_metric = 'rmse' 
                                )
        model.fit(train_x, train_y, 
              eval_set=[(train_x, train_y),(valid_x, valid_y)], 
              early_stopping_rounds=100, verbose=100)   
        joblib.dump(model, "xgbm_"+str(i)+".m")
        print("xgb_model_%d has saved"%i)
    return model
def get_train_model(df_, m, m_type='lgb',i=0):
    df = df_.copy()
    # 数据集划分m=25,26,27,28,
    st = 13#start time 
    all_idx   = (df['mt'].between(st , m-1))#原版
    train_idx = (df['mt'].between(st , m-5))
    valid_idx = (df['mt'].between(m-4, m-4))
    test_idx  = (df['mt'].between(m  , m  ))
    # 最终确认
    train_x = df[train_idx][features]
    train_y = df[train_idx]['n_label']
    valid_x = df[valid_idx][features]
    valid_y = df[valid_idx]['n_label']   
    # get model
    model = get_model_type(train_x,train_y,valid_x,valid_y,m_type,i)  
    # offline
    df['pred_label'] = np.expm1(model.predict(df[features]))
    best_score = score(df[valid_idx]) 
    # online
    if m_type == 'lgb':
        model.n_estimators = model.best_iteration_ + 100
        model.fit(df[all_idx][features], df[all_idx]['n_label'], categorical_feature=cate_feat)
    elif m_type == 'xgb':
        model.n_estimators = model.best_iteration + 100
        model.fit(df[all_idx][features], df[all_idx]['n_label'])
    df['forecastVolum'] = np.expm1(model.predict(df[features]))
    print('valid mean:',df[valid_idx]['pred_label'].mean())
    print('true  mean:',df[valid_idx]['label'].mean())
    print('test  mean:',df[test_idx]['forecastVolum'].mean())
    # 阶段结果
    sub = df[test_idx][['id']]
    sub['forecastVolum'] = df[test_idx]['forecastVolum'].apply(lambda x: 0 if x < 0 else x).round().astype(int) 
    print(sub.shape)
    return sub,df[valid_idx]['pred_label']
for month in [25,26,27,28]: 
    m_type = 'lgb' 
    data['n_label'] = np.log1p(data['label'])
    data_df, stat_feat = get_stat_feature(data)#每次都要更新下特征
    num_feat = ['regYear'] + stat_feat
    cate_feat = ['adcode','bodyType','model','regMonth',]#,'k_mean_1','k_mean'
    if m_type == 'lgb':
        for i in cate_feat:
            data_df[i] = data_df[i].astype('category')
    elif m_type == 'xgb':
        lbl = LabelEncoder()  
        for i in tqdm(cate_feat):
            data_df[i] = lbl.fit_transform(data_df[i].astype(str)) 
    features = num_feat + cate_feat
    print(len(features), len(set(features)))   
    sub,val_pred = get_train_model(data_df, month, m_type,month-24)   
    data.loc[(data.regMonth==(month-24))&(data.regYear==2018), 'salesVolume'] = sub['forecastVolum'].values
    data.loc[(data.regMonth==(month-24))&(data.regYear==2018), 'label'      ] = sub['forecastVolum'].values
sub = data.loc[(data.regMonth>=1)&(data.regYear==2018), ['id','salesVolume']]
sub.columns = ['id','forecastVolum']
sub[['id','forecastVolum']].round().astype(int).to_csv('../input/B_res.csv', index=False)
#结果基于规则纠正
my_data=pd.read_csv('../input/B_res.csv')
my_data["forecastVolum"]=my_data["forecastVolum"]*0.79-5
my_data["forecastVolum"]=(my_data["forecastVolum"]).astype(int)
my_data.loc[my_data[my_data["forecastVolum"] < 4].index,"forecastVolum"]=4
my_data.loc[my_data[my_data["forecastVolum"] >9000].index,"forecastVolum"]=9000
my_data.to_csv('../input/submit.csv',index=0)