训练集数据分析
# 导入相关包
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
plt.rc('font', family='SimHei', size=13)
import os
import gc
import re
import warnings
import sys
warnings.filterwarnings("ignore")
path = './data/'
# ------------------------读取数据---------------------
#####train
trn_click = pd.read_csv(path+'train_click_log.csv')
item_df = pd.read_csv(path+'articles.csv')
item_df = item_df.rename(columns={'article_id': 'click_article_id'}) #重命名,方便后续match
item_emb_df = pd.read_csv(path+'articles_emb.csv')
#####test
tst_click = pd.read_csv(path+'testA_click_log.csv')
# ----------------------数据预处理------------------------------
# 对每个用户的点击时间戳进行排序
trn_click['rank'] = trn_click.groupby(['user_id'])['click_timestamp'].rank(ascending=False).astype(int)
tst_click['rank'] = tst_click.groupby(['user_id'])['click_timestamp'].rank(ascending=False).astype(int)
#计算用户点击文章的次数,并添加新的一列count
trn_click['click_cnts'] = trn_click.groupby(['user_id'])['click_timestamp'].transform('count')
tst_click['click_cnts'] = tst_click.groupby(['user_id'])['click_timestamp'].transform('count')
# ----------------------数据浏览-----------------------------
trn_click = trn_click.merge(item_df, how='left', on=['click_article_id'])
print(trn_click.head(5))
#用户点击日志信息
print(trn_click.info())
print(trn_click.describe())
#训练集中的用户数量为20w
print(trn_click.user_id.nunique())
print(trn_click.groupby('user_id')['click_article_id'].count().min()) # 训练集里面每个用户至少点击了两篇文章)
# 画直方图大体看一下基本的属性分布
plt.figure()
plt.figure(figsize=(15, 20))
i = 1
for col in ['click_article_id', 'click_timestamp', 'click_environment', 'click_deviceGroup', 'click_os', 'click_country', 'click_region', 'click_referrer_type', 'rank', 'click_cnts']:
plot_envs = plt.subplot(5, 2, i)
i += 1
v = trn_click[col].value_counts().reset_index()[:10]
fig = sns.barplot(x=v['index'], y=v[col])
for item in fig.get_xticklabels():
item.set_rotation(90)
plt.title(col)
plt.tight_layout()
plt.show()
测试集用户点击情况
# 导入相关包
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
plt.rc('font', family='SimHei', size=13)
import os
import gc
import re
import warnings
import sys
warnings.filterwarnings("ignore")
path = './data/'
# ------------------------读取数据---------------------
#####train
trn_click = pd.read_csv(path+'train_click_log.csv')
item_df = pd.read_csv(path+'articles.csv')
item_df = item_df.rename(columns={'article_id': 'click_article_id'}) #重命名,方便后续match
item_emb_df = pd.read_csv(path+'articles_emb.csv')
#####test
tst_click = pd.read_csv(path+'testA_click_log.csv')
# ----------------------数据预处理------------------------------
# 对每个用户的点击时间戳进行排序
trn_click['rank'] = trn_click.groupby(['user_id'])['click_timestamp'].rank(ascending=False).astype(int)
tst_click['rank'] = tst_click.groupby(['user_id'])['click_timestamp'].rank(ascending=False).astype(int)
#计算用户点击文章的次数,并添加新的一列count
trn_click['click_cnts'] = trn_click.groupby(['user_id'])['click_timestamp'].transform('count')
tst_click['click_cnts'] = tst_click.groupby(['user_id'])['click_timestamp'].transform('count')
# ----------------------数据浏览-----------------------------
trn_click = trn_click.merge(item_df, how='left', on=['click_article_id'])
tst_click = tst_click.merge(item_df, how='left', on=['click_article_id'])
user_click_merge = trn_click.append(tst_click)
# -------------------用户点击环境变化分析-----------------
def plot_envs(df, cols, r, c):
plt.figure()
plt.figure(figsize=(10, 5))
i = 1
for col in cols:
plt.subplot(r, c, i)
i += 1
v = df[col].value_counts().reset_index()
fig = sns.barplot(x=v['index'], y=v[col])
for item in fig.get_xticklabels():
item.set_rotation(90)
plt.title(col)
plt.tight_layout()
plt.show()
# 分析用户点击环境变化是否明显,这里随机采样10个用户分析这些用户的点击环境分布
sample_user_ids = np.random.choice(tst_click['user_id'].unique(), size=5, replace=False)
sample_users = user_click_merge[user_click_merge['user_id'].isin(sample_user_ids)]
cols = ['click_environment','click_deviceGroup', 'click_os', 'click_country', 'click_region','click_referrer_type']
for _, user_df in sample_users.groupby('user_id'):
plot_envs(user_df, cols, 2, 3)
用户点击新闻数量的分布
用户活跃度
点击次数排前50的用户的点击次数都在100次以上,我们可以定义点击次数大于等于100次的用户为活跃用户,
而小于等于两次的用户非常的多,这些用户可以认为是非活跃用户
新闻点击次数分析
可以看出点击次数最多的前100篇新闻,点击次数大于1000次
点击次数最多的前20篇新闻,点击次数大于2500可以定义为热门新闻
可以发现很多新闻只被点击过一两次,可以定义这些新闻是冷门新闻。