spark：协同过滤--49

上一周了解了一下协同过滤在spark中的应用，但是找了几本书也没找到具体的算法流程······每天像只没头的苍蝇一样东一头西一头乱飞···每天室友们睡觉之后才能效率高一点

协同过滤算法：

常被用于推荐系统，MLlib支持基于模型的协同过滤，使用交替最小二乘法（ALS）

1.显示的用户反馈

2.隐示的用户反馈

参数：

numBlocks：用于并行化计算的分块个数（设置为-1时自动配置）

rank：模型中隐性因子的个数

iterations：迭代次数

lambda：是ALS的正则规划参数

implicitPrefs：显示反馈ALS版本或隐式版本

alpha：一个针对隐性反馈ALS版本的参数

一。推荐引擎根据不同用户的行为，推荐不同的数据

二。推荐引擎的数据源

    1.用户的基本信息------基于人口统计学推荐

    2.物品或内容的相关性------基于内容推荐

    3.用户对物品或信息偏好------基于协同过滤推荐

三。推荐模型的建立方式

    1.基于物品和用户本身的推荐

        a。基于用户的协同过滤推荐（相同爱好的人）

        b。基于项目的协同过滤推荐（功能相似---物品）

    2.基于关联规则的推荐

    3.基于模型的推荐（构造一个推荐模型）

Amazon最早使用推荐模型，豆瓣最早将推荐模型用于社交网站

///

package llf

import org.apache.log4j.{Level, Logger}
import org.apache.spark.mllib.recommendation.{MatrixFactorizationModel, Rating, ALS}
import org.apache.spark.{SparkContext, SparkConf}
import org.apache.spark.SparkContext._
import scala.collection.mutable.ListBuffer
import scala.io.Source
import org.apache.spark.rdd._
import java.io.File
/**
 * Created by sendoh on 2015/6/11.
 */
object MovieLensALS {
  def main(args: Array[String]): Unit ={
    //屏蔽日志
    Logger.getLogger("org.apache.spark").setLevel(Level.WARN)
    Logger.getLogger("org.eclipse.jetty.server").setLevel(Level.OFF)
    //
    if (args.length != 3){
      println("Usage: java -jar code.jar dependency_jars file_location save_location")
      System.exit(0)
    }
    val jars = ListBuffer[String]()
    args(0).split(',').map(jars += _)
    //运行环境
    val conf = new SparkConf().setAppName("MovieLensALS").setMaster("local[3]").setSparkHome("/usr/local/spark-1.2.0-bin-hadoop2.4").setJars(jars)
    val sc = new SparkContext(conf)
    //装载用户评分，该评分由评分器产生
    val myRatings = loadRatings("hdfs://localhost:9000/datatnt/MovieTest/ratings.dat")
    val myRatingsRDD = sc.parallelize(myRatings, 1)
    //装载用户评分文件
    def loadRatings(path: String): Seq[Rating] = {
      val lines = Source.fromFile(path).getLines()
      val ratings = lines.map{ line =>
        val fields = line.split("::")
        Rating(fields(0).toInt, fields(1).toInt, fields(2).toDouble)
      }.filter(_.rating > 0.0)
      if (ratings.isEmpty){
        sys.error("NO ratings provided")
      } else {
        ratings.toSeq
      }
    }
    //样本数据目录
    //装载样本评分数据，其中最后一列Timestamp取除10的余数作为key，Rating为值，即（Int, RAting）
    val ratings = sc.textFile("hdfs://localhost:9000/datatnt/MovieTest/ratings.dat").map{ line =>
      val fields = line.split("::")
      (fields(3).toLong % 10, Rating(fields(0).toInt, fields(1).toInt, fields(2).toDouble))
    }
    //装载电影目录对照表(电影ID -> 电影标题)
    val movies = sc.textFile("hdfs://localhost:9000/datatnt/MovieTest/movies.dat").map{ line =>
      val fields = line.split("::")
      (fields(0).toInt, fields(1))
    }.collect().toMap

    val numRatings = ratings.count()
    val numUsers = ratings.map(_._2.user).distinct().count()
    val numMovies = ratings.map(_._2.product).distinct().count()
    println("Got " + numRatings + " ratings from " + numUsers + " users on " + numMovies + " movies.")
    //将样本评分表以key值切分成3个部分，分别用于训练（60%，并加入用户评分），校验（20%），测试（20%）
    //该数据在计算过程中要多次用到，cache到内存
    val numPartitions = 4
    val training = ratings.filter(x => x._1 < 6).values.union(myRatingsRDD).repartition(numPartitions).cache()
    val validation = ratings.filter(x => x._1 > 6 && x._1 < 8).values.repartition(numPartitions).cache()
    val test = ratings.filter(x => x._1 >= 8).values.cache()

    val numTraining = training.count()
    val numValidation = validation.count()
    val numTest = test.count()
    println("Training: " + numTraining + ", validation: " + numValidation + ", test: " + numTest)

    //训练不同参数下的模型，并在校验集中验证，获取最佳参数下的模型
    val ranks = List(8, 9)
    val lambdas = List(0.1, 10.0)
    val numIters = List(10, 20)
    var bestModel: Option[MatrixFactorizationModel] = None
    var bestValidationRmse = Double.MaxValue
    var bestRank = 0
    var bestLambda = -1.0
    var bestNumIter = -1
    for (rank <- ranks; lambda <- lambdas; numIter <- numIters){
      val model = ALS.train(training, rank, numIter, lambda)
      val validationRmse = computeRmse(model, validation, numValidation)
      println("RMSE (validation) = " + validationRmse + " for the model trained with rank = " + rank + ", lambda = " + lambda +
        ", and numIter = " + numIter + ".")
      if (validationRmse < bestValidationRmse){
        bestModel = Some(model)
        bestValidationRmse = validationRmse
        bestRank = rank
        bestLambda = lambda
        bestNumIter = numIter
      }
    }

    //校验集预测数据和实际数据之间的均方根差
    def computeRmse(model: MatrixFactorizationModel, data: RDD[Rating], n: Long): Double = {
      val predictions: RDD[Rating] = model.predict(data.map(x => (x.user, x.product)))
      val predictionsAndRatings = predictions.map(x => ((x.user, x.product), x.rating)).join(data.map(x => ((x.user, x.product), x.rating))).values
      math.sqrt(predictionsAndRatings.map(x => (x._1 - x._2) * (x._1 - x._2)).reduce(_ + _) / n)
    }

    //用最佳模型预测测试集的评分，并计算和实际评分之间的均方根误差
    val testRmse = computeRmse(bestModel.get, test, numTest)
    println("The best model was trained with rank = " + bestRank + " and lambda = " + bestLambda + ", and numIter = " + bestNumIter
      + ", and its RMSE on the test set is " + testRmse + ".")
    //创建一个本机基本模型和与最好的模型进行比较
    val meanRating = training.union(validation).map(_.rating).mean
    val baselineRmse = math.sqrt(test.map(x => (meanRating - x.rating) * (meanRating - x.rating)).mean)
    val improvement = (baselineRmse - testRmse) / baselineRmse * 100
    println("The best model improves the baseline by " + "%1.2f".format(improvement) + "%.")

    //推荐前十部最感兴趣的电影，注意要剔除用户已经评分的电影
    val myRatedMovieIds = myRatings.map(_.product).toSet
    val candidates = sc.parallelize(movies.keys.filter(!myRatedMovieIds.contains(_)).toSeq)
    val recommendations = bestModel.get.predict(candidates.map((0, _))).collect().sortBy(-_.rating).take(10)
    var i = 1
    println("Movies recommended for you: ")
    recommendations.foreach { r =>
      println("%2d".format(i) + ": " + movies(r.product))
      i += 1
    }
    sc.stop()
  }

}