天池新闻推荐入门赛之数据分析

训练集数据分析

# 导入相关包

import pandas as pd
import numpy as np

import matplotlib.pyplot as plt
import seaborn as sns
plt.rc('font', family='SimHei', size=13)
import os
import gc
import re
import warnings
import sys
warnings.filterwarnings("ignore")

path = './data/'

# ------------------------读取数据---------------------
#####train
trn_click = pd.read_csv(path+'train_click_log.csv')
item_df = pd.read_csv(path+'articles.csv')
item_df = item_df.rename(columns={'article_id': 'click_article_id'})  #重命名，方便后续match
item_emb_df = pd.read_csv(path+'articles_emb.csv')

#####test
tst_click = pd.read_csv(path+'testA_click_log.csv')


# ----------------------数据预处理------------------------------
# 对每个用户的点击时间戳进行排序
trn_click['rank'] = trn_click.groupby(['user_id'])['click_timestamp'].rank(ascending=False).astype(int)
tst_click['rank'] = tst_click.groupby(['user_id'])['click_timestamp'].rank(ascending=False).astype(int)

#计算用户点击文章的次数，并添加新的一列count
trn_click['click_cnts'] = trn_click.groupby(['user_id'])['click_timestamp'].transform('count')
tst_click['click_cnts'] = tst_click.groupby(['user_id'])['click_timestamp'].transform('count')

# ----------------------数据浏览-----------------------------
trn_click = trn_click.merge(item_df, how='left', on=['click_article_id'])
print(trn_click.head(5))
#用户点击日志信息
print(trn_click.info())
print(trn_click.describe())

#训练集中的用户数量为20w
print(trn_click.user_id.nunique())

print(trn_click.groupby('user_id')['click_article_id'].count().min())  # 训练集里面每个用户至少点击了两篇文章)

# 画直方图大体看一下基本的属性分布
plt.figure()
plt.figure(figsize=(15, 20))
i = 1
for col in ['click_article_id', 'click_timestamp', 'click_environment', 'click_deviceGroup', 'click_os', 'click_country', 'click_region', 'click_referrer_type', 'rank', 'click_cnts']:
    plot_envs = plt.subplot(5, 2, i)
    i += 1
    v = trn_click[col].value_counts().reset_index()[:10]
    fig = sns.barplot(x=v['index'], y=v[col])
    for item in fig.get_xticklabels():
        item.set_rotation(90)
    plt.title(col)
plt.tight_layout()
plt.show()

测试集用户点击情况

# 导入相关包

import pandas as pd
import numpy as np

import matplotlib.pyplot as plt
import seaborn as sns
plt.rc('font', family='SimHei', size=13)
import os
import gc
import re
import warnings
import sys
warnings.filterwarnings("ignore")

path = './data/'

# ------------------------读取数据---------------------
#####train
trn_click = pd.read_csv(path+'train_click_log.csv')
item_df = pd.read_csv(path+'articles.csv')
item_df = item_df.rename(columns={'article_id': 'click_article_id'})  #重命名，方便后续match
item_emb_df = pd.read_csv(path+'articles_emb.csv')

#####test
tst_click = pd.read_csv(path+'testA_click_log.csv')


# ----------------------数据预处理------------------------------
# 对每个用户的点击时间戳进行排序
trn_click['rank'] = trn_click.groupby(['user_id'])['click_timestamp'].rank(ascending=False).astype(int)
tst_click['rank'] = tst_click.groupby(['user_id'])['click_timestamp'].rank(ascending=False).astype(int)

#计算用户点击文章的次数，并添加新的一列count
trn_click['click_cnts'] = trn_click.groupby(['user_id'])['click_timestamp'].transform('count')
tst_click['click_cnts'] = tst_click.groupby(['user_id'])['click_timestamp'].transform('count')

# ----------------------数据浏览-----------------------------
trn_click = trn_click.merge(item_df, how='left', on=['click_article_id'])
tst_click = tst_click.merge(item_df, how='left', on=['click_article_id'])

user_click_merge = trn_click.append(tst_click)
# -------------------用户点击环境变化分析-----------------
def plot_envs(df, cols, r, c):
    plt.figure()
    plt.figure(figsize=(10, 5))
    i = 1
    for col in cols:
        plt.subplot(r, c, i)
        i += 1
        v = df[col].value_counts().reset_index()
        fig = sns.barplot(x=v['index'], y=v[col])
        for item in fig.get_xticklabels():
            item.set_rotation(90)
        plt.title(col)
    plt.tight_layout()
    plt.show()


# 分析用户点击环境变化是否明显，这里随机采样10个用户分析这些用户的点击环境分布
sample_user_ids = np.random.choice(tst_click['user_id'].unique(), size=5, replace=False)
sample_users = user_click_merge[user_click_merge['user_id'].isin(sample_user_ids)]
cols = ['click_environment','click_deviceGroup', 'click_os', 'click_country', 'click_region','click_referrer_type']
for _, user_df in sample_users.groupby('user_id'):
    plot_envs(user_df, cols, 2, 3)

用户点击新闻数量的分布

用户活跃度

点击次数排前50的用户的点击次数都在100次以上，我们可以定义点击次数大于等于100次的用户为活跃用户，

而小于等于两次的用户非常的多，这些用户可以认为是非活跃用户

新闻点击次数分析

可以看出点击次数最多的前100篇新闻，点击次数大于1000次

点击次数最多的前20篇新闻，点击次数大于2500可以定义为热门新闻

可以发现很多新闻只被点击过一两次，可以定义这些新闻是冷门新闻。