TF-IDF计算相似文章

第一步：使用jieba分词注册成udf，给每个帖子分词。

%spark_recommend.pyspark
from pyspark.sql.types import BooleanType,LongType
from scipy.stats import norm, t
from pyspark.sql import SparkSession, DataFrame, functions as F
from pyspark.sql.functions import udf,lit
from dateutil.parser import parse
import string
from pyspark.sql.types import IntegerType, ArrayType
import pandas as pd
import pymysql
import jieba
import re
from pyspark.sql.functions import udf
from pyspark.sql.types import ArrayType, StringType
from pyspark.sql.functions import col,udf, json_tuple, lit

dp='2020-04-20'
start_date='2020-04-20'
end_date='2020-04-20'

STOP_WORDS =set(['的', '之', '吗','了'])
def is_chinese(content):
    """检查是否是纯中文"""
    return re.compile('^[\u4e00-\u9fa5]*$').match(content) is not None
def cut_post(content):
    # """分词且只保留中文单词。
    return [i for i in jieba.cut(content) if is_chinese(i) and i not in STOP_WORDS]

spark = SparkSession.builder \
    .appName("cold_start_decay_group_1") \
    .enableHiveSupport() \
    .getOrCreate()
    
tb_post = 'dwd.dwd_post_fact'

# 步骤一：取最近600天的帖子
# 步骤二：计算热帖：最近30点击数大于2
# 步骤三：热帖 union c类帖，然后distinct
# 步骤四：注册分词udf（分词-过滤非中文-过滤停用词），返回ArrayType(StringType())类型。

df = spark.table(tb_post)
df = df.filter(df.post_type.isin([2, 5, 8])).filter(df.display_yn == 1)
df = df.filter('content is not null')
df = df.filter(df.create_date >=start_date)
df = df.selectExpr('post_id', 'content', 'create_date', 'post_video_yn','tags.name tags', 'uid')
assert df.count() > 1000, '表%s数据源出问题了，帖子数量应该大于1000条' % tb_post

post =df.distinct()
spark.udf.register('cut_udf', cut_post)
post =post.selectExpr('post_id', 'create_date', 'cut_udf(content) as words')
post.show()
# word_cnt = post.select(F.explode('words').alias('word')).distinct().count()
# print('去重后词总数为%d'% word_cnt)
# post.write.mode('overwrite').saveAsTable('tmp.post_words3') 

处理后的数据：

第二步：使用SparkML的生成帖子的tf·idf的稀疏矩阵。

 

from pyspark.sql import SparkSession, functions as F
from pyspark.ml.feature import HashingTF, IDF, Normalizer
from scipy.sparse import csr_matrix  # 稀疏矩阵
import sys

spark = SparkSession.builder.appName("search_statistics").getOrCreate()
log = spark.read.format("csv").option("header","true").option("inferSchema", True).load("data-after-cut.csv")

NumFeatures = 259498

print("现有的dataframe结构")
log.printSchema()
log.show(3)
#string转成array
word_df =log.withColumn("words", F.split(log["words"], ","))\
    .withColumn("tags", F.split(log["tags"], ","))

#使用HashingTF得到 tf，然后传入idf.fit(tf)得到id_model。  把tf传入idf_model，就能得到所有idf_vec
hashing_tf = HashingTF(inputCol="words", outputCol="tf",numFeatures=NumFeatures)
tf_df = hashing_tf.transform(word_df)
idf = IDF(minDocFreq=5, inputCol="tf", outputCol="sv")
idf_model = idf.fit(tf_df)
hashing_tf.write().overwrite().save('tf')
idf_model.write().overwrite().save('idf')
idf_df = idf_model.transform(tf_df)
# 向量归一化
normalizer = Normalizer(p=2.0, inputCol="sv", outputCol="target_vec") 
target_df = normalizer.transform(idf_df).drop('sv', 'tf').filter('target_vec is not null')
print('过滤之前%d' % target_df.count())
new_post = target_df.filter(F.size(target_df.words) > 10)
print('过滤之后%d' % new_post.count())
targets = new_post.select('post_id', 'target_vec').collect()
negative_feeds = [68982,78237] ##黑名单
id_list = []  # 合格post_id
svs = []  # 合格post的ti-idf向量
for _id, sv in targets:
    if _id in negative_feeds:
        continue
    id_list.append(_id)
    svs.append(sv)
print('目标post_id', id_list[: 10])
print('目标post的ti-idf向量', svs[:3])
""" 构建目标post的稀疏矩阵(词索引, if-idf值)
    文章1 [if-idf, if-idf, 0,]
    文章2 [if-idf, 0, if-idf,]
    文章3 [0, if-idf, if-idf,]
"""
row, col, data = [], [], []
for i, sv in enumerate(svs):
    indices = sv.indices
    row.extend([i] * len(indices))
    col.extend(indices)
    data.extend(sv.values)
mat = csr_matrix((data, (row, col)), shape=(len(svs), NumFeatures))
print('mat系数矩阵', mat[:10])

第三步：注册UDF（计算当前item和tf-idf矩阵每个item的余弦相似度，堆排序取top 10）

from pyspark.sql.types import StructType, StructField, LongType,FloatType, ArrayType
import heapq

REC_TYPE = StructType([
    StructField("post_id", LongType(), False),
    StructField('score', FloatType(), True)
])

def __item_rec(mat, id_list, n_rec, cur_post_id, sv):
    """返回与 v 余弦相似度最大的 n 个物品id及相似度。
    NOTE: 推荐输出数量可能会小于n_rec，比如一共四个帖子，用户已经看过2个，
    因为会过滤用户看过的帖子，所以最多只能推出两个。
    :param mat: {list of SparseVector}  物品向量矩阵
    :param id_list: {list} 所有的post_id，用于将索引转为post_id
    :param n_rec: {int} TopN推荐的数量
    :param cur_post_id: {int} 当前的post_id
    :param sv: {Vector} 当前物品向量
    :return: [(id, score), (id, score)...]
    """
    cosine_sims = mat.dot(sv).tolist()
    recs = (i for i in zip(id_list, cosine_sims) if i[0] != cur_post_id)
    return heapq.nlargest(n_rec, recs, key=lambda x: x[1])

n_rec=10
related_rec = F.udf(lambda post_id, v: __item_rec(mat, id_list, n_rec, post_id, v), ArrayType(REC_TYPE))
df = target_df.select('post_id', related_rec('post_id', 'target_vec').alias('recs')).cache()
df.show() #字符串截断：字符串最大长度180

说明一下：得到帖子A和帖子B的的 tf·idf向量， 然后cos= a·b/(|a|*|b|)，因为a和b都是经过归一化，所以|a|和|b|都是1，cos就是直接计算向量的点积，即直接调用dot方法即可。