此项目的目的是搭建一个简易的新闻推荐系统, 采用的数据集为英国BBC新闻。需要经历的几个步骤是：
- 文本数据的预处理
- 通过新闻的tf-idf词率，并使用主题模型来给每一个新闻抽取主题的分布
- 通过用户历史新闻的浏览记录，并建立用户关注的主题分布，并为用户做推荐
# make sure these libs are imported
from gensim import corpora, models, parsing
from glob import glob
import warnings
import os,sys
import re,string
import pickle
import nltk
from nltk.corpus import stopwords
from nltk.stem.snowball import SnowballStemmer
from nltk.stem import WordNetLemmatizer# 只考虑新闻长度大于如下给定值的情况
min_length = 200
# 主题数量, 超参, 需要根据情况调整，代表的是针对于每一个新闻要提取多少个主题。
num_topics = 90
# 设置一个单词出现在数据集中的最小次数, 小于这个次数的单词会不予考虑,
# 这是为了减小总的单词数, 出现次数太小的单词对主题的贡献可以忽略不计
no_below_this_number = 50
# 设置一个单词出现在数据集中的最大次数, 如果单词出现在文章中的次数大
# 于文章总数乘以如下百分比, 则不予考虑. 这是因为太频繁出现在多个文章
# 中的单词很可能为Stop_Words, 例如is, am, of, at等, 它们对主题
# 的贡献也可以忽略不计
no_above_fraction_of_doc = 0.2
# 如果一个主题的权重低于如下值, 则将其从主题中去除, 说明这个主题很
# 不重要, 需要按照实际情况调整。
remove_topic_so_less = 0.05
# 推断语料库的主题分布时，最大迭代次数。数据集越小, 需要迭代次数
# 越大, 需要按照实际情况调整。
num_of_iteration = 1000
# 训练的迭代次数, 即遍历数据集的次数, 需要按照实际情况调整
passes = 3
# stop_words 停用词
stopword = stopwords.words('english')1. 数据的处理： 对数据集进行tokenization
pathToCorpora = "BBCNews"
print('Start tokenzing the corpora: %s' % (pathToCorpora))
punct = re.compile('[%s]' % re.escape(string.punctuation))
wnl = WordNetLemmatizer() # 使用基于worldnet的lemmzation操作
# 新闻的序号, 从0开始
doc_count=0
train_set = []
doc_mapping = {}
link_mapping = {}
for f in glob(pathToCorpora+'/*'):
filereader = open(f, 'r',encoding='utf-8')
article = filereader.readlines();
filereader.close()
text = ''
try: # 抽取新闻中的链接，标题以及文本
link = article[0]
title = article[1]
text = article[2].lower()
except IndexError:
continue
# 如果新闻长度太小，不考虑
if len(text) < min_length:
continue
# 去掉新闻中的标点符号
text = punct.sub("",text)
# 将每一条新闻转为单词集合
tokens = nltk.word_tokenize(text)
# 从单词中去除 stopword, 然后对单词进行Lemmatize (例如将单词 go, goes, going, gone 都转为 go)
# 并添加到训练数据集train_set中
train_sub_set = [wnl.lemmatize(word) for word in tokens if word not in stopword]
train_set.append(train_sub_set)
# 将文章的标题加入doc_mapping字典, 使用文章出现的序号为字典的key
doc_mapping[doc_count] = title
# 将文章的http链接加入link_mapping字典, 使用文章出现的序号为字典的key
link_mapping[doc_count] = link
# 为新闻的序号添加 1
doc_count = doc_count+1
# 每处理 500 篇文献, 打印处理的新闻数量
if doc_count % 500 == 0:
print( 'Have processed %i documents' % (doc_count))# 将训练数据放入 gensim.corpora 的字典中, 为后续的训练做准备
dic = corpora.Dictionary(train_set)
# 从字典中去除出现频次太多或者太少的单词
denominator = len(dic)
dic.filter_extremes(no_below=no_below_this_number, no_above=no_above_fraction_of_doc)
nominator = len(dic)
# 将训练数据中的单词转换为词袋(bag of words)的方式
corpus = [dic.doc2bow(text) for text in train_set]
print( '数据集一共包含 %i 条新闻' % (doc_count))
print( "原来的字典中一共包含 ", denominator, " 个单词")
print( "去除频次异常的单词后, 一共有 ", nominator, " 个单词, 减少的百分比为 ", (1-(nominator/denominator)),"%")
print( '完成数据的清洗')2. 训练主题模型
# 将训练数据中的单词从词袋(bag of words)的表达方式转为Tfidf(Term Frequency, Inverse Document Frequency)的方式
tfidf = models.TfidfModel(corpus)
corpus_tfidf = tfidf[corpus]
# 训练主题模型, 调用合适的函数，并传入参数，试图了解一下参数的含义
lda = models.LdaModel(corpus=corpus, id2word=dic, num_topics=90)
corpus_lda = lda[corpus_tfidf]
# 训练结束以后, 打印所有的主题及及其相关最频繁的单词
for i in range(num_topics):
print( '主题 %s : ' % (str(i)) + lda.print_topic(i))
# 计算当前主题模型对应的 perplexity. 其值越小说明主题模型越好
print( '===============================')
print( '主题模型的 perplexity : ',lda.bound(corpus_lda),' 主题数量为 k =', str(num_topics))
print( '===============================')# 定义辅助函数, 将列表转为字典到
def __convertListToDict(anylist):
convertedDict = {}
for pair in anylist:
topic = pair[0]
weight = pair[1]
convertedDict[topic] = weight
return convertedDict
# 保存python对象到文件中
def __savePickleFile(fileName,objectName):
fileName= './LDAmodel/'+fileName+'.pickle'
mappingFile = open(fileName,'wb')
pickle.dump(objectName,mappingFile)
mappingFile.close()
# 保存LDA模型
save_path = './LDAmodel/final_ldamodel'
lda.save(save_path)
# 保存数据集
save_path = 'corpus'
__savePickleFile(save_path,corpus)
# 保存序号到文章标题的字典
save_path = 'documentmapping'
__savePickleFile(save_path,doc_mapping)
# 保存序号到文章链接的字典
save_path = 'linkmapping'
__savePickleFile(save_path,link_mapping)
# 保存文章序号到主题分布的字典
doc_topic_matrix = {}
count = 0
for doc in corpus:
dense_vector = {}
vector = __convertListToDict(lda[doc])
# 去除权重太低的主题
for topic in vector:
if vector[topic] > remove_topic_so_less:
dense_vector[topic] = vector[topic]
doc_topic_matrix[count] = dense_vector
count = count+1
save_path = 'doc_topic_matrix'
__savePickleFile(save_path,doc_topic_matrix)
3. 使用主题模型做推荐¶
假设摸一个用户历史上阅读过的问题序号为 10, 21, 31, 我们需要为此用户推荐新闻。
# 从保存的模型文件中加载模型
path_doc_topic_matrix = './LDAmodel/doc_topic_matrix.pickle'
mappingFile = open(path_doc_topic_matrix,'rb')
doc_topic_matrix = pickle.load(mappingFile)
mappingFile.close()
# 加载文章序号到新闻标题的字典
path_mappingfile= './LDAmodel/documentmapping.pickle'
mappingFile = open(path_mappingfile,'rb')
mapping = pickle.load(mappingFile)
mappingFile.close()
# 加载文章序号到新闻链接的字典
path_mappingfile= './LDAmodel/linkMapping.pickle'
mappingFile = open(path_mappingfile,'rb')
linkMapping = pickle.load(mappingFile)
mappingFile.close()
# 假设摸一个用户历史上阅读过的新闻序号为 10, 21, 31, 我们为此用户计算其新闻喜好的主题模型
# 建立的方式是将其阅读过的新闻的主题的权重相加并求均值
user_dict = {}
for i in [10, 21, 31]:
user_dict[i] = doc_topic_matrix[i]
omit_topic_below_this_fraction = 0.1
user_topic_vector = {}
length = len(user_dict)
for seen_doc in user_dict:
for seen_topic in user_dict[seen_doc]:
weight = user_dict[seen_doc][seen_topic]
if seen_topic in user_topic_vector:
current_weight = user_topic_vector[seen_topic]
current_weight = current_weight + weight/length
user_topic_vector[seen_topic] = current_weight
else:
user_topic_vector[seen_topic] = weight/length
# 去除权重低于 omit_topic_below_this_fraction/2 的主题
normalized_user_topic_vector = {}
for k,v in user_topic_vector.items():
if v > omit_topic_below_this_fraction/2:
normalized_user_topic_vector[k] = v
# 对主题分布做归一化, 使得所有权重之和为 1
# 最后的输出为： 当前用户感兴趣的新闻主题分布 : {5: 0.6319591628063931, 11: 0.2349756591871135, 80: 0.1330651780064935}
#**********************************************
# ！！！！！结果有些问题！！！！！
values = []
for k,v in normalized_user_topic_vector.items():
values.append(v)
for k,v in normalized_user_topic_vector.items():
normalized_user_topic_vector[k] = v/sum(values)
print( '当前用户感兴趣的新闻主题分布 : {0}'.format(normalized_user_topic_vector))# 使用 Pearson correlation 计算每一个文章的主题分布和用户感兴趣的主题分布的相似性
def fill_list_from_dict(a,topics):
result = [0] * topics
for k,v in a.items():
result[k-1] = v
return result
def pearson_correlation(a,b,topics):
from scipy.stats import pearsonr
a = fill_list_from_dict(a,topics)
b = fill_list_from_dict(b,topics)
return pearsonr(a,b)[0]
recommend_dict={}
for doc in doc_topic_matrix:
sim = pearson_correlation(normalized_user_topic_vector, doc_topic_matrix[doc], lda.num_topics)
if doc not in user_dict.keys():
recommend_dict[doc] = sim
# 为用户推荐的新闻数量
def getOrderedDict(dic):
from operator import itemgetter
from collections import OrderedDict
sorteddict = OrderedDict(sorted(dic.items(), key=itemgetter(1),reverse=True))
return sorteddict
no_of_recommendation = 5
sort = getOrderedDict(recommend_dict)
recommend_str = (str(list(sort.keys())[:no_of_recommendation])
.replace('[','')
.replace(']','')
)
print('===============用户读过的新闻===========')
for i in user_dict:
print('id: {0}, 标题: {1}, 链接: {2}'.format(i, mapping[i], linkMapping[i]))
print("===============为用户推荐的新闻=========")
for i in list(sort.keys())[:no_of_recommendation]:
print('id: {0}, 标题: {1}, 链接: {2}'.format(i, mapping[i], linkMapping[i]))
验证结果：
===============用户读过的新闻===========
id: 10, 标题: 314 mobile phones 'stolen in London every day'
, 链接: http://www.bbc.com/news/uk-england-london-21018569
id: 21, 标题: 3D printing to rebuild patient's face at Morriston Hospital
, 链接: http://www.bbc.com/news/uk-wales-south-west-wales-24926598
id: 31, 标题: 3D-printed canal home takes shape in Amsterdam
, 链接: http://www.bbc.com/news/technology-22152212
===============为用户推荐的新闻=========
id: 3503, 标题: Architect plans 3D-printed buildings
, 链接: http://www.bbc.com/news/technology-21121061
id: 4242, 标题: Astronauts on the International Space Station have to use a computer system far more cumbersome and f
, 链接: http://www.bbc.com/future/story/20140417-how-to-reboot-a-space-station
id: 3925, 标题: Art research effort aided by face recognition
, 链接: http://www.bbc.com/news/technology-22764822
id: 3996, 标题: As fuel prices soar and cities become more congested, car manufacturers ' and the public - are beginn
, 链接: http://www.bbc.com/future/story/20120521-electric-dreams
id: 1064, 标题: After eating asparagus, some people can detect a strange smell, while others claim not to notice a th
, 链接: http://www.bbc.com/future/story/20140818-mystery-of-asparagus-and-urine
1568
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
