LightGBM 模型上线Pipeline

1. 模型预测Pipeline

1. model_predict_pipeline.sh

1. hive表拉取预测数据
2. 预测数据切分
3. 模型打分
4. 打分数据整合
5. 数据上传

# 1.从hive表中拉去预测数据到本地（模型工程目录）
/app/hadoop/hive/bin/hive -e "
use db_name;
set hive.cli.print.header=false;
set hive.resultset.use.unique.column.names=false;
select phone
,flag
,feature1
,feature2
,feature3
,feature4
,feature5
from table_name
;
" > predict_data.csv

FILE_PATH="/data/zz/pull_new/algo1"
# 2.将预测数据（体量较大几十个G，避免一次性加载到内存，速度慢）切分成小文件
# 清空分割后小文件存放目录
rm $FILE_PATH/data/*
# 每5KW条拆分成一个小文件
split -l 50000000  predict_data.csv $FILE_PATH/data/new_ --verbose
# 清空模型打分小文件存放目录
rm $FILE_PATH/output/*
# 3. 模型打分
/data/zz/zz_venv/bin/python model_predict.py $FILE_PATH/data/
# 模型打分小文件合并成一个文件 
cat $FILE_PATH/output/new_*predict_output.csv >$FILE_PATH/model_score_output.csv
 
# 判断以上是否执行成功，成功才上传数据，否退出并返回-1
if [ $? -eq 0 ]
then
    echo "model predict success"
else
    echo "---------model predict fail-----------"
    return -1
fi

4. 模型打分数据导入到hive表
/app/hadoop/hive/bin/hive -e "
load data local inpath '$FILE_PATH/model_score_output.csv' overwrite
into table table_name partition (pt='${last_dt}');
alter table table_name drop partition(pt<'${last_7dt}');
"
 
echo "-----done!"
 

model_ predict.py

# coding:utf-8
import pandas as pd
import lightgbm as lgb
import sys
import os
  
THRESHOLD = 0.1245
lgb_model_file = 'algo1.model'
lgb_model = lgb.Booster(model_file=lgb_model_file)
file_path = "/data/zz/pull_new/algo1/output/"

# hive 表中的字段
all_names = ['phone', 'flag', 'feature1', 'feature2', 'feature3', 'feature4', 'feature5']
# 用作特征的字段    
feature_names = ['feature1', 'feature2', 'feature3', 'feature4', 'feature5']
# 结果文件保留的字段   
keep_cols = ['phone', 'score', 'flag']

def predict(pathname, f):
    filename = pathname + f
    test = pd.read_csv(filename, sep = '\t', header=None)
    test.columns = all_names
    test['score'] = lgb_model.predict(test.loc[:, feature_names])
    test = test.loc[test['score'] >= THRESHOLD, keep_cols]
    # shuffle
    test = test.sample(frac=1, random_state=1024)
    test.to_csv(file_path+f+"_predict_output.csv", index=