O2O使用券

最新推荐文章于 2020-04-23 10:43:00 发布

weixin_43534779

最新推荐文章于 2020-04-23 10:43:00 发布

阅读量1k

点赞数

本文链接：https://blog.csdn.net/weixin_43534779/article/details/84921210

版权

import os, sys, pickle

import numpy as np
import pandas as pd

import matplotlib.pyplot as plt
import matplotlib.dates as mdates

import seaborn as sns

from datetime import date

from sklearn.model_selection import KFold, train_test_split, StratifiedKFold, cross_val_score, GridSearchCV
from sklearn.pipeline import Pipeline
from sklearn.linear_model import SGDClassifier, LogisticRegression
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import log_loss, roc_auc_score, auc, roc_curve
from sklearn.preprocessing import MinMaxScaler

import xgboost as xgb
# import lightgbm as lgb

数据分析

# 数据读取
dfoff = pd.read_csv(r'C:\Users\dell\Desktop\o2o使用券\ccf_offline_stage1_train.csv',engine = 'python')
dftest = pd.read_csv(r'C:\Users\dell\Desktop\o2o使用券\ccf_offline_stage1_test_revised.csv',engine = 'python')
dfon = pd.read_csv(r'C:\Users\dell\Desktop\o2o使用券\ccf_online_stage1_train.csv',engine = 'python')
print('data read end.')

data read end.

dfoff.head()#查看前五行数据

	User_id	Merchant_id	Coupon_id	Discount_rate	Distance	Date_received	Date
0	1439408	2632	NaN	NaN	0.0	NaN	20160217.0
1	1439408	4663	11002.0	150:20	1.0	20160528.0	NaN
2	1439408	2632	8591.0	20:1	0.0	20160217.0	NaN
3	1439408	2632	1078.0	20:1	0.0	20160319.0	NaN
4	1439408	2632	8591.0	20:1	0.0	20160613.0	NaN

dfoff.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 1754884 entries, 0 to 1754883
Data columns (total 7 columns):
User_id          int64
Merchant_id      int64
Coupon_id        float64
Discount_rate    object
Distance         float64
Date_received    float64
Date             float64
dtypes: float64(4), int64(2), object(1)
memory usage: 93.7+ MB

# print('有优惠券，购买商品条数', dfoff[(dfoff['Date_received'] != 'nan') & (dfoff['Date'] != 'nan')].shape[0])
# print('无优惠券，购买商品条数', dfoff[(dfoff['Date_received'] == 'nan') & (dfoff['Date'] != 'nan')].shape[0])
# print('有优惠券，不购买商品条数', dfoff[(dfoff['Date_received'] != 'nan') & (dfoff['Date'] == 'nan')].shape[0])
# print('无优惠券，不购买商品条数', dfoff[(dfoff['Date_received'] == 'nan') & (dfoff['Date'] == 'nan')].shape[0])
print('有优惠券，购买商品条数', dfoff[(pd.notnull(dfoff['Date_received'])) & (pd.notnull(dfoff['Date']))].shape[0])
print('无优惠券，购买商品条数',  dfoff[(pd.isnull(dfoff['Date_received'])) & (pd.notnull(dfoff['Date']))].shape[0])
print('有优惠券，不购买商品条数',  dfoff[(pd.notnull(dfoff['Date_received'])) & (pd.isnull(dfoff['Date']))].shape[0])
print('无优惠券，不购买商品条数', dfoff[(pd.isnull(dfoff['Date_received'])) & (pd.isnull(dfoff['Date']))].shape[0])

有优惠券，购买商品条数 75382
无优惠券，购买商品条数 701602
有优惠券，不购买商品条数 977900
无优惠券，不购买商品条数 0

# 在测试集中出现的用户但训练集没有出现
print('1. User_id in training set but not in test set', set(dftest['User_id']) - set(dfoff['User_id']))
# 在测试集中出现的商户但训练集没有出现
print('2. Merchant_id in training set but not in test set', set(dftest['Merchant_id']) - set(dfoff['Merchant_id']))

1. User_id in training set but not in test set {2495873, 1286474}
2. Merchant_id in training set but not in test set {5920}

优惠券和距离特征

print('Discount_rate类型：',dfoff['Discount_rate'].unique())

Discount_rate类型： [nan '150:20' '20:1' '200:20' '30:5' '50:10' '10:5' '100:10' '200:30'
 '20:5' '30:10' '50:5' '150:10' '100:30' '200:50' '100:50' '300:30'
 '50:20' '0.9' '10:1' '30:1' '0.95' '100:5' '5:1' '100:20' '0.8' '50:1'
 '200:10' '300:20' '100:1' '150:30' '300:50' '20:10' '0.85' '0.6' '150:50'
 '0.75' '0.5' '200:5' '0.7' '30:20' '300:10' '0.2' '50:30' '200:100'
 '150:5']

print('Distance类型：',dfoff['Distance'].unique())

Distance类型： [ 0.  1. nan  2. 10.  4.  7.  9.  3.  5.  6.  8.]

把 xx:bb,类型转换成比率： 1 - bb/xx ，同时建立折扣券相关的特征：discount_rate,discount_man,discount_jian,discount_type

# 1. 将满xx减yy类型(`xx:yy`)的券变成折扣率 : `1 - yy/xx`，同时建立折扣券相关的特征 `discount_rate, discount_man, discount_jian, discount_type`
# 2. 将距离 `str` 转为 `int`
# convert Discount_rate and Distance
def getDiscountType(row):
    if pd.isnull(row):
        return np.nan
    elif ':' in row:
        return 1
    else:
        return 0

def convertRate(row):
    """Convert discount to rate"""
    if pd.isnull(row):
        return 1.0
    elif ':' in str(row):
        rows = row.split(':')
        return 1.0 - float(rows[1])/float(rows[0])
    else:
        return float(row)

def getDiscountMan(row):
    if ':' in str(row):
        rows = row.split(':')
        return int(rows[0])
    else:
        return 0

def getDiscountJian(row):
    if ':' in str(row):
        rows = row.split(':')
        return int(rows[1])
    else:
        return 0
print("tool is ok.")

tool is ok.

def processData(df):
    # convert discunt_rate
    df['discount_rate'] = df['Discount_rate'].apply(convertRate)
    df['discount_man'] = df['Discount_rate'].apply(getDiscountMan)
    df['discount_jian'] = df['Discount_rate'].apply(getDiscountJian)
    df['discount_type'] = df['Discount_rate'].apply(getDiscountType)
    print(df['discount_rate'].unique())
    # convert distance
    df['distance'] = df['Distance'].fillna(-1).astype(int)
    print(df['distance'].unique())
    return df

dfoff = processData(dfoff)
dftest = processData(dftest)

[1.         0.86666667 0.95       0.9        0.83333333 0.8
 0.5        0.85       0.75       0.66666667 0.93333333 0.7
 0.6        0.96666667 0.98       0.99       0.975      0.33333333
 0.2        0.4       ]
[ 0  1 -1  2 10  4  7  9  3  5  6  8]
[0.83333333 0.9        0.96666667 0.8        0.95       0.75
 0.98       0.5        0.86666667 0.6        0.66666667 0.7
 0.85       0.33333333 0.94       0.93333333 0.975      0.99      ]
[ 1 -1  5  2  0 10  3  6  7  4  9  8]

dfoff.head(2)

	User_id	Merchant_id	Coupon_id	Discount_rate	Distance	Date_received	Date	discount_rate	discount_man	discount_jian	discount_type	distance
0	1439408	2632	NaN	NaN	0.0	NaN	20160217.0	1.000000	0	0	NaN	0
1	1439408	4663	11002.0	150:20	1.0	20160528.0	NaN	0.866667	150	20	1.0	1

dftest.head(2)

	User_id	Merchant_id	Coupon_id	Discount_rate	Distance	Date_received	discount_rate	discount_man	discount_jian	discount_type	distance
0	4129537	450	9983	30:05:00	1.0	20160712	0.833333	30	5	1	1
1	6949378	1300	3429	30:05:00	NaN	20160706	0.833333	30	5	1	-1

时间处理：优惠券获得时间，优惠券使用时间

# date_received = dfoff['Date_received'].unique()
# date_received = sorted(date_received[date_received != 'null'])

# date_buy = dfoff['Date'].unique()
# date_buy = sorted(date_buy[date_buy !='null'])

# # date_buy = sorted(dfoff[dfoff['Date'] != 'null']['Date'])
# date_buy = sorted(dfoff[dfoff['Date'] != 'null']['Date'])
# # print()

# print('优惠券收到日期从',date_received[0],'到', date_received[-1])
# print('消费日期从', date_buy[0], '到', date_buy[-1])

couponbydate = dfoff[dfoff['Date_received'].notna()][['Date_received', 'Date']].groupby(['Date_received'], as_index=False).count()
# print(couponbydate.head())
couponbydate.columns = ['Date_received','count']
print(couponbydate.head())
buybydate = dfoff[(dfoff['Date'].notna()) & (dfoff['Date_received'].notna())][['Date_received', 'Date']].groupby(['Date_received'], as_index=False).count()
buybydate.columns = ['Date_received','count']

# dfoff['Date_received'].notna()

   Date_received  count
0     20160101.0     74
1     20160102.0     67
2     20160103.0     74
3     20160104.0     98
4     20160105.0    107

# sns.set_style('ticks')
# sns.set_context("notebook", font_scale= 1.4)
# plt.figure(figsize = (12,8))
# date_received_dt = pd.to_datetime('Date_received', format='%Y%m%d %H')
# # date_received_dt = pd.to_datetime('Date_received')

# plt.subplot(211)
# plt.bar(date_received_dt, couponbydate['count'], label = 'number of coupon received' )
# plt.bar(date_received_dt, buybydate['count'], label = 'number of coupon used')
# plt.yscale('log')
# plt.ylabel('Count')
# plt.legend()

# plt.subplot(212)
# plt.bar(date_received_dt, buybydate['count']/couponbydate['count'])
# plt.ylabel('Ratio(coupon used/coupon received)')
# plt.tight_layout()

新建关于星期的特征

weekday {1,2,3,4,5,6,7}
周一到周五为 0 ，周六日为 1

def getWeekday(row):
    if row == 'nan': 
        return np.nan
    else:
        return date(int(row[0:4]), int(row[4:6]),int(row[6:8])).weekday() +1

dfoff['weekday'] = dfoff['Date_received'].astype(str).apply(getWeekday)
dftest['weekday'] = dftest['Date_received'].astype(str).apply(getWeekday)

dfoff['weekday_type'] = dfoff['weekday'].apply(lambda x : 1  if x in [6,7] else 0 )
dftest['weekday_type'] = dftest['weekday'].apply(lambda x : 1  if x in [6,7] else 0 )

weekdaycols = ['weekday_' + str(i) for i in range(1,8)]
print(weekdaycols)

['weekday_1', 'weekday_2', 'weekday_3', 'weekday_4', 'weekday_5', 'weekday_6', 'weekday_7']

tmpdf = pd.get_dummies(dfoff['weekday'].replace('null', np.nan))
tmpdf1 = pd.get_dummies(dftest['weekday'].replace('null', np.nan))

tmpdf.columns = weekdaycols
tmpdf1.columns = weekdaycols

dfoff[weekdaycols] = tmpdf
dftest[weekdaycols] = tmpdf1

dfoff[weekdaycols].head()

dftest[weekdaycols].head()
# dfoff.info()

	weekday_2	weekday_3	weekday_5
0	1	0	0
1	0	1	0
2	0	1	0
3	0	1	0
4	0	0	1

对收到优惠券的行进行分类

Date != ‘null’ & Date-Date_received <= 15: y = 1
Date_received == ‘null’: y = -1
other : y =0


def label(row):
    if pd.isnull(row['Date_received']):
        return -1
    if pd.notnull(row['Date']):
        td = pd.to_datetime(row['Date'], format='%Y%m%d') -  pd.to_datetime(row['Date_received'], format='%Y%m%d')
        if td <= pd.Timedelta(15, 'D'):
            return 1
    return 0

dfoff['label'] = dfoff.apply(label, axis = 1)

print("end")

print(dfoff['label'].value_counts())

 0    988887
-1    701602
 1     64395
Name: label, dtype: int64

print('已有columns：',dfoff.columns.tolist())

已有columns： ['User_id', 'Merchant_id', 'Coupon_id', 'Discount_rate', 'Distance', 'Date_received', 'Date', 'discount_rate', 'discount_man', 'discount_jian', 'discount_type', 'distance', 'weekday', 'weekday_type', 'weekday_1', 'weekday_2', 'weekday_3', 'weekday_4', 'weekday_5', 'weekday_6', 'weekday_7', 'label']

dfoff.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 1754884 entries, 0 to 1754883
Data columns (total 22 columns):
User_id          int64
Merchant_id      int64
Coupon_id        float64
Discount_rate    object
Distance         float64
Date_received    float64
Date             float64
discount_rate    float64
discount_man     int64
discount_jian    int64
discount_type    float64
distance         int32
weekday          float64
weekday_type     int64
weekday_1        uint8
weekday_2        uint8
weekday_3        uint8
weekday_4        uint8
weekday_5        uint8
weekday_6        uint8
weekday_7        uint8
label            int64
dtypes: float64(7), int32(1), int64(6), object(1), uint8(7)
memory usage: 205.9+ MB

dfoff.head()

	User_id	Merchant_id	Coupon_id	Discount_rate	Distance	Date_received	Date	discount_rate	discount_man	discount_jian	...	weekday	weekday_type	weekday_1	weekday_3	weekday_6	label
0	1439408	2632	NaN	NaN	0.0	NaN	20160217.0	1.000000	0	0	...	NaN	0	0	0	0	-1
1	1439408	4663	11002.0	150:20	1.0	20160528.0	NaN	0.866667	150	20	...	6.0	1	0	0	1	0
2	1439408	2632	8591.0	20:1	0.0	20160217.0	NaN	0.950000	20	1	...	3.0	0	0	1	0	0
3	1439408	2632	1078.0	20:1	0.0	20160319.0	NaN	0.950000	20	1	...	6.0	1	0	0	1	0
4	1439408	2632	8591.0	20:1	0.0	20160613.0	NaN	0.950000	20	1	...	1.0	0	1	0	0	0

5 rows × 22 columns

dfoff.count()

User_id          1754884
Merchant_id      1754884
Coupon_id        1053282
Discount_rate    1053282
Distance         1648881
Date_received    1053282
Date              776984
discount_rate    1754884
discount_man     1754884
discount_jian    1754884
discount_type    1053282
distance         1754884
weekday          1053282
weekday_type     1754884
weekday_1        1754884
weekday_2        1754884
weekday_3        1754884
weekday_4        1754884
weekday_5        1754884
weekday_6        1754884
weekday_7        1754884
label            1754884
dtype: int64

# data split
df = dfoff[dfoff['label'] != -1].copy()
train = df[(df['Date_received'] < 20160516)].copy()
valid = df[(df['Date_received'] >= 20160516) & (df['Date_received'] <= 20160615)].copy()
print(train['label'].value_counts())
print(valid['label'].value_counts())

0    759172
1     41524
Name: label, dtype: int64
0    229715
1     22871
Name: label, dtype: int64

# feature
original_feature = ['discount_rate','discount_type','discount_man', 'discount_jian','distance', 'weekday', 'weekday_type'] + weekdaycols
print(len(original_feature),original_feature)

14 ['discount_rate', 'discount_type', 'discount_man', 'discount_jian', 'distance', 'weekday', 'weekday_type', 'weekday_1', 'weekday_2', 'weekday_3', 'weekday_4', 'weekday_5', 'weekday_6', 'weekday_7']

# model1
predictors = original_feature
print(predictors)

def check_model(data, predictors):
    
    classifier = lambda: SGDClassifier(
        loss='log', 
        penalty='elasticnet', 
        fit_intercept=True, 
        max_iter=100, 
        shuffle=True, 
        n_jobs=1,
        class_weight=None)

    model = Pipeline(steps=[
        ('ss', StandardScaler()),
        ('en', classifier())
    ])

    parameters = {
        'en__alpha': [ 0.001, 0.01, 0.1],
        'en__l1_ratio': [ 0.001, 0.01, 0.1]
    }

    folder = StratifiedKFold(n_splits=3, shuffle=True)
    
    grid_search = GridSearchCV(
        model, 
        parameters, 
        cv=folder, 
        n_jobs=-1, 
        verbose=1)
    grid_search = grid_search.fit(data[predictors], 
                                  data['label'])
    
    return grid_search

if not os.path.isfile('1_model.pkl'):
    model = check_model(train, predictors)
    print(model.best_score_)
    print(model.best_params_)
    with open('1_model.pkl', 'wb') as f:
        pickle.dump(model, f)
else:
    with open('1_model.pkl', 'rb') as f:
        model = pickle.load(f)

['discount_rate', 'discount_type', 'discount_man', 'discount_jian', 'distance', 'weekday', 'weekday_type', 'weekday_1', 'weekday_2', 'weekday_3', 'weekday_4', 'weekday_5', 'weekday_6', 'weekday_7']

# valid predict
y_valid_pred = model.predict_proba(valid[predictors])
valid1 = valid.copy()
valid1['pred_prob'] = y_valid_pred[:, 1]
valid1.head(2)

	User_id	Merchant_id	Coupon_id	Discount_rate	Distance	Date_received	Date	discount_rate	discount_man	discount_jian	...	weekday_type	weekday_1	weekday_2	weekday_3	weekday_4	weekday_5	weekday_6	weekday_7	label	pred_prob
1	1439408	4663	11002.0	150:20	1.0	20160528.0	NaN	0.866667	150	20	...	1	0	0	0	0	0	1	0	0	0.018534
4	1439408	2632	8591.0	20:1	0.0	20160613.0	NaN	0.950000	20	1	...	0	1	0	0	0	0	0	0	0	0.140446

2 rows × 23 columns

# avgAUC calculation
vg = valid1.groupby(['Coupon_id'])
aucs = []
for i in vg:
    tmpdf = i[1] 
    if len(tmpdf['label'].unique()) != 2:
        continue
    fpr, tpr, thresholds = roc_curve(tmpdf['label'], tmpdf['pred_prob'], pos_label=1)
    aucs.append(auc(fpr, tpr))
print(np.average(aucs))

0.5351686740665353

print("----train-----")
model = SGDClassifier(#lambda:
    loss='log',
    penalty='elasticnet',
    fit_intercept=True,
    max_iter=100,
    shuffle=True,
    alpha = 0.01,
    l1_ratio = 0.01,
    n_jobs=1,
    class_weight=None
)
model.fit(train[original_feature], train['label'])

----train-----





SGDClassifier(alpha=0.01, average=False, class_weight=None, epsilon=0.1,
       eta0=0.0, fit_intercept=True, l1_ratio=0.01,
       learning_rate='optimal', loss='log', max_iter=100, n_iter=None,
       n_jobs=1, penalty='elasticnet', power_t=0.5, random_state=None,
       shuffle=True, tol=None, verbose=0, warm_start=False)

# #### 预测以及结果评价
print(model.score(valid[original_feature], valid['label']))

0.9094526220772331

# test prediction for submission
y_test_pred = model.predict_proba(dftest[original_feature])
dftest1 = dftest[['User_id','Coupon_id','Date_received']].copy()
dftest1['label'] = y_test_pred[:,1]
dftest1.to_csv('submit2.csv', index=False, header=False)
dftest1.head()

	User_id	Coupon_id	Date_received	label
0	4129537	9983	20160712	0.096343
1	6949378	3429	20160706	0.139846
2	2166529	6928	20160727	0.002833
3	2166529	1808	20160727	0.013445
4	6172162	6500	20160708	0.070036

weixin_43534779

关注

0
点赞
踩
1

收藏

觉得还不错? 一键收藏
0
评论
O2O使用券

import os, sys, pickleimport numpy as npimport pandas as pdimport matplotlib.pyplot as pltimport matplotlib.dates as mdatesimport seaborn as snsfrom datetime import datefrom sklearn.model_...
复制链接

扫一扫