:scala版算法实现
package com.bbw5.dataalgorithms.spark
import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
import scala.collection.mutable.ArrayBuffer
/**
* finds all association rules
* for a market basket data sets.
*
* author:baibaiw5
*/
object SparkFindAssociationRules {
def main(args: Array[String]) {
val sparkConf = new SparkConf().setAppName("SparkFindAssociationRules")
val sc = new SparkContext(sparkConf)
//支持度
val support = sc.broadcast(1)
//置信度
val confidence = sc.broadcast(0.01)
val rawData = sc.parallelize(Seq("A, C", "B, D", "A, C, E", "C, E", "A, B, E", "B, E"), 2)
//sort the items
val data = rawData.map { _.split(",").map { _.trim() }.sorted.toList }
data.foreach(println)
// generate frequent patterns
val patData = data.flatMap { a =>
val items = new ArrayBuffer[List[String]]
//gen all kinds of item pair
for (i <- 1 to a.size) {
items ++= a.combinations(i)
}
items.map { a => (a, 1) }
}.reduceByKey((a, b) => a + b).filter(a => a._2 >= support.value)
patData.foreach(println)
//generate all sub - patterns
val subPatData = patData.flatMap { a =>
val (pat, count) = a
val items = new ArrayBuffer[(List[String], (List[String], Int))]
items += pat -> (List[String]("null") -> count)
//as least 2 items
items ++= pat.map(a => pat.diff(List(a))).filter(a => a.size >= 1).map { b => b -> a }
items
}.groupByKey()
subPatData.foreach(println)
//Generate Association Rules
val assoRuleData = subPatData.flatMap { a =>
val items = new ArrayBuffer[(List[String], List[String], Double)]
val (pat, list) = a
val patCount = list.filter(p => p._1(0) == "null").head._2
items ++= list.filter(p => p._1(0) != "null").map(f => (pat, f._1.diff(pat), f._2 / patCount.toDouble))
items.filter(p => p._3 >= confidence.value)
items
}
assoRuleData.foreach(println)
}
}