PySpark对电影和用户进行聚类分析

之前的博文使用pyspark.mllib.recommendation做推荐案例，代码实现了如何为用户做电影推荐和为电影找到潜在的感兴趣用户。本篇博文介绍如何利用因子分解出的用户特征、电影特征做用户和电影的聚类分析，以看能否找到不同于已知的、有趣的新信息。

第一步：获取用户评分数据显式因式分解后的movieFactors、userFactors。

from pyspark.mllib.recommendation import ALS,Rating
#用户评分数据
rawData = sc.textFile("/Users/gao/data/ml-100k/u.data")
rawRatings = rawData.map(lambda line:line.split("\t")[:3])
#构造user-item-rating 数据
ratings = rawRatings.map(lambda line:Rating(user=int(line[0]),product=int(line[1]),rating=float(line[2])))
#评分数据放入缓存中
ratings.cache()

#模型训练
alsModel = ALS.train(ratings, rank=50, iterations=10, lambda_=0.1) 

#根据因式分解出的 productFeatures\userFeatures 
from pyspark.mllib.linalg import Vectors
movieFactors = alsModel.productFeatures().map(lambda product_features:(product_features[0],Vectors.dense(product_features[1])))
movieVectors = movieFactors.map(lambda line:line[1])
userFactors = alsModel.userFeatures().map(lambda user_features:(user_features[0],Vectors.dense(user_features[1])))
userVectors = userFactors.map(lambda line:line[1])

第二步，读入电影题材数据

#电影题材数据
genres = sc.textFile("/Users/gao/Desktop/Toby/5Spark-JDK/data/ml-100k/u.genre")

#电影题材数据按照 | 分割
genreMap = genres.filter(lambda line:line!='' and len(line)>4).map(lambda line:line.split('|')).map(lambda arr:(arr[1],arr[0])).collectAsMap()
print(genreMap)

#电影名称和题材标注数据
movies = sc.textFile("/Users/gao/data/ml-100k/u.item")

#获取电影名称和题材名对应数据
import numpy as np 
def  titles_and_genres(line,genreMap):
	arr = line.split('|')
	idx = int(arr[0])
	title =  str(arr[1])
	genre_list = np.array([int(i) for i in arr[5:]])
	index_list = [index for index in np.where(genre_list==1)[0]]
	genresAssigned = [genreMap[str(index)] for index in index_list]
	print(genresAssigned)
	return (idx,(title,genresAssigned))

genreMap_bcast = sc.broadcast(genreMap)
titlesAndGenres = movies.map(lambda line:titles_and_genres(line,genreMap_bcast.value))

第三步，查看特征数据是否需要规范化

#查看特征数据是否需要规范化
from pyspark.mllib.linalg.distributed import RowMatrix
movieMatrix = RowMatrix(movieVectors)
movieMatrixSummary =movieMatrix.computeColumnSummaryStatistics()
userMatrix = RowMatrix(userVectors)
userMatrixSummary =userMatrix.computeColumnSummaryStatistics()
print("Movie factors mean: {}".format(movieMatrixSummary.mean()))
print("Movie factors variance:  {}".format(movieMatrixSummary.variance()))
print("User factors mean:  {}".format(userMatrixSummary.mean()))
print("User factors variance:  {}".format(userMatrixSummary.variance()))

第四步，训练模型-KMeans聚类

from pyspark.mllib.clustering import KMeans
numClusters = 5
numIterations = 10
#训练产品特征数据的聚类 ，查看下聚类中心
movieClusterModel = KMeans.train(rdd=movieVectors, k=numClusters,maxIterations=numIterations)
movieClusterModel.clusterCenters

#预测聚类中心
predictions = movieClusterModel.predict(movieVectors)

第五步，解释聚类含义

#解释聚类的含义
import math
def computeDistance(v1,v2):
	v = v1-v2
	return v.dot(v)

titlesWithFactors = titlesAndGenres.join(movieFactors)

#查看电影的聚类情况
def movieAssignedCluster(line):
	(id,((title,genres), vector)) = line
	pred = movieClusterModel.predict(vector)
	clusterCentre = movieClusterModel.clusterCenters[pred]
	dist = computeDistance(Vectors.dense(clusterCentre),Vectors.dense(vector))
	return (id, title, genres, pred, dist)

moviesAssigned = titlesWithFactors.map(lambda line:movieAssignedCluster(line))

clusterAssignments = moviesAssigned.groupBy(lambda x:x[3]).collectAsMap()

#输出每个聚类中，距离聚类中心最近的Top电影
for (k,v) in clusterAssignments.items():
	print('Cluster %d:'%k)
	m = sorted([(x[1],x[2],x[4]) for x in v], key=lambda x:x[-1], reverse=False) 
	print([x for x in m[:20]])
	print("==========")

聚类0中的电影基本是Drama类型的，偏向爱情、浪漫的题材。

第六步，拆分训练集、测试集，调聚类中心参数K，并评价聚类模型

#拆分训练集、测试集
trainTestSplitMovies = movieVectors.randomSplit(np.array([0.6, 0.4]),123)
trainMovies = trainTestSplitMovies[0]
testMovies = trainTestSplitMovies[1]
print('Movie clustering cross-validation:')
#调参-聚类中心K
for k in [2, 3, 4, 5, 10, 20,30,40,50]:
	model = KMeans.train(rdd=trainMovies, k=k,maxIterations=numIterations)
	cost = model.computeCost(testMovies)
	print('WCSS for k={} is {:.4f}'.format(k,cost))

随着K的增大，WCSS减小，但到K=20后，WCSS的减小趋势变缓。所以，结合需求的同时，K取在20以内比较好。

Done