使用CTR评估

使用LR模型对CTR进行预估

import os
import sys
# 如果当前代码文件运行测试需要加入修改路径，避免出现后导包问题
BASE_DIR = os.path.dirname(os.path.dirname(os.getcwd()))
sys.path.insert(0, os.path.join(BASE_DIR))

PYSPARK_PYTHON = "/miniconda2/envs/reco_sys/bin/python"
# 当存在多个版本时，不指定很可能会导致出错
os.environ["PYSPARK_PYTHON"] = PYSPARK_PYTHON
os.environ["PYSPARK_DRIVER_PYTHON"] = PYSPARK_PYTHON

from pyspark.ml.feature import OneHotEncoder
from pyspark.ml.feature import StringIndexer
from pyspark.ml import Pipeline
from pyspark.sql.types import *
from pyspark.ml.feature import VectorAssembler
from pyspark.ml.classification import LogisticRegression
from pyspark.ml.classification import LogisticRegressionModel
from offline import SparkSessionBase

class CtrLogisticRegression(SparkSessionBase):

    SPARK_APP_NAME = "ctrLogisticRegression"
    ENABLE_HIVE_SUPPORT = True

    def __init__(self):

        self.spark = self._create_spark_hbase()

ctr = CtrLogisticRegression()

进行行为日志数据读取

#- (1)进行行为日志数据读取
ctr.spark.sql('use profile')
user_article_basic = ctr.spark.sql("select * from user_article_basic").select(['user_id', 'article_id', 'clicked'])
user_article_basic.show()

用户画像读取处理与日志数据合并

#- (2)用户画像读取处理与日志数据合并
user_profile_hbase = ctr.spark.sql("select user_id, information.gender, information.birthday, article_partial from user_profile_hbase")
user_profile_hbase