import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
object Topdata10 {
def main(args: Array[String]): Unit = {
val sparConf: SparkConf = new SparkConf().setMaster("local[4]").setAppName("Top10")
val sc: SparkContext = new SparkContext(sparConf)
val actionRDD: RDD[String] = sc.textFile("data/user_visit_action.txt")
val value: RDD[(String, (Int, Int, Int))] = actionRDD.flatMap(action => {
val data2: Array[String] = action.split("_")
if (data2(6) != "-1") {
List((data2(6), (1, 0, 0)))}
else if (data2(8) != "null") {
val da2 = data2(8).split(",")
da2.map(id => (id, (0, 1, 0)))}
else if (data2(10) != "null") {
val da2 = data2(10).split(",")
da2.map(id => (id, (0, 0, 1)))}
else {Nil}})
val value1: RDD[(String, (Int, Int, Int))] = value.reduceByKey((t1, t2) => {
(t1._1 + t2._1, t1._2 + t2._2, t1._3 + t2._3)})
val tuples: Array[(String, (Int, Int, Int))] = value1.sortBy(_._2, false).take(10)
tuples.foreach(println)
sc.stop()}
}
Spark 练习题-Top10热门品类
最新推荐文章于 2024-09-03 21:03:55 发布