传递方法名选择不同的方法,_ :占位符
object RecdOfCui {
private val decimalFormatter = new DecimalFormat()
decimalFormatter.setMaximumFractionDigits(3)
def main(args: Array[String]) {
Logger.getLogger("org.apache.spark").setLevel(Level.WARN)
Logger.getLogger("org.apache.eclipse.jetty.server").setLevel(Level.OFF)
calculateSimilarity(tvRDD, diceSimilarity(_, _))
.map(f => f._1 + "\t" + f._2 + "\t" + decimalFormatter.format(f._3))
.saveAsTextFile()
calculateSimilarity(tvRDD, labelSimilarity(_, _))
.map(f => f._1 + "\t" + f._2 + "\t" + decimalFormatter.format(f._3))
.saveAsTextFile()
sc.stop()
}
def calculateSimilarity(video: RDD[(String, Set[String])]
, myFunc: (Set[String], Set[String]) => Double):
RDD[(String, String, Double)] = {
video.cartesian(video)
.map(f => (f._1._1, f._2._1, myFunc(f._1._2, f._2._2)))
.filter(f => f._3 > 0).groupBy(_._1)
.take(100))
}
def diceSimilarity(set1: Set[String], set2: Set[String]): Double = {
if (set1.isEmpty | set2.isEmpty) 0
else {
val togetherSet = set1 & set2
if (togetherSet.isEmpty) 0
else 2 * togetherSet.size / (set1.size + set2.size).toDouble
}
}
def labelSimilarity(set1: Set[String], set2: Set[String]): Double = {
if (set1.isEmpty | set2.isEmpty) 0
else {
val bingJiSize = (set1 & set2).size
if (bingJiSize == 0) 0
else bingJiSize / (set1.size + set2.size - bingJiSize).toDouble
}
}
}