Spark-core电商分析

1.需求一：Top10热门品类

2.​​​​​​​需求2：Top10热门品类中每个品类的Top10活跃Session统计

3.需求3：页面单跳转化率统计

Spark-core电商分析

需求一代码

package demo

import org.apache.spark.broadcast.Broadcast
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
import org.junit.Test
/*
  二次排序

  思路：
  1.统计点击类别的数量并排序
  2.将前十类别放到广播变量中
  3.获取 点击类别列，Sessionsid列
  4.根据广播变量过滤掉不是前十的类别
  5.获取结果((类别,Session),1)
  6.reduceByKey得出((类别,Session),sum)
  7.map得出(类别,(Session,sum)),此时的类别只剩前十类别的
  8.只对值(Session,sum)进行排序然后去前十
  9.mapValues(f=>f.toList.sortBy(f=>f._2).take(10))
 */
class demo电商分析 {
  @Test
  def methods_1(): Unit ={
    val conf: SparkConf = new SparkConf().setMaster("local[*]").setAppName("pro_1")
    val sc = new SparkContext(conf)
    val rdd: RDD[String] = sc.textFile(" user_visit_action.txt", 2)
    val rdd1: RDD[Array[String]] = rdd.map(f => f.split("_"))
    val value = rdd1.map(f => (f(0), f(1), f(2), f(3), f(4), f(5), f(6),
      f(7), f(8), f(9), f(10), f(11), f(12)))
    value.cache()//第二问会用该值
    //求点击前十类别
    val tuples: Array[(String, Int)] = value.filter(f => f._7.toInt != -1)
      .map(f => f._7)
      .groupBy(f => f)
      .map(f => (f._1, f._2.size))
      .sortBy(f => f._2, false)
      .take(10)
      //tuples.foreach(f=>println(f))  //输出
    println("-------------------------------------------------------------")

    //运用了广播变量,用于保留前十的类别，类似mr中求join的思路
    val bc: Broadcast[Array[String]] = sc.broadcast(tuples.map(f => f._1).toArray)

    //思路：将数据转换为（（cid,session）,1）再进行聚合，
    //聚合后进行转化为（cid，（session，sum））来按cid分组按sum排序取top1
    value.filter(f=>f._7.toInt != -1 && bc.value.contains(f._7))//运用广播变量获取前十的类别数据集
      .map(f=>((f._7,f._3),1))
      .reduceByKey((v1,v2)=>v1+v2) //聚合
      .map(f=>(f._1._1,(f._1._2,f._2)))
      .groupByKey()
      //给二层元组进行排序
      //法1
      // .map(f=>(f._1,f._2.toList.sortBy(f=>f._2).take(10)))
      //法2
      .mapValues(f=>f.toList.sortBy(f=>f._2).take(10))
      .foreach(f=>println("品类编号："+f._1,"SessionId："+f._2))

  }
}

需求二代码

UserVisitAction

package homework.homework4

//实体类
case class UserVisitAction(
                 var          date: String, //用户点击行为的日期
                 var    user_id: Long, //用户的ID
                 var    session_id: String, //Session的ID
                 var    page_id: Long, //某个页面的ID
                 var    action_time: String, //动作的时间点
                 var    search_keyword: String, //用户搜索的关键词
                 var    click_category_id: Long, //某一个商品品类的ID
                 var    click_product_id: Long, //某一个商品的ID
                 var    order_category_ids: String, //一次订单中所有品类的ID集合
                 var    order_product_ids: String,   //一次订单中所有商品的ID集合
                 var    pay_category_ids: String,    //一次支付中所有品类的ID集合
                 var    pay_product_ids: String,     //一次支付中所有商品的ID集合
                 var    city_id: Long) //城市 id
case  class  totalcount(
                      var id:String,
                      var  clinkcount:Long,
                      var ordercount:Long,
                      var  paycount:Long
                       )

package homework.homework4

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
import org.junit.Test

import scala.collection.mutable.ListBuffer
//分别统计top10类别点击的次数，下单的次数，支付的次数
/*
  思路：
  1.读取文件，Map进行分割，封装到对象中
  2.flatMap进行一行一行的处理，判断点击、下单、支付
  3.对下单、支付的id进行循环，添加到数组并返回 （totalcount）
  4.按键分组，只对值进行累加
*/
class 电商分析2 {
  //返回对象 UserVisitAction集合
  @Test
  def mainMethod():Unit={
    val conf: SparkConf = new SparkConf().setMaster("local[*]").setAppName("pro_1")
    val sc = new SparkContext(conf)
    val rdd: RDD[String] = sc.textFile(" user_visit_action.txt", 2)

    val value: RDD[UserVisitAction] = rdd.map(data => {
      val f: Array[String] = data.split("_")
      //封装
      UserVisitAction(
        f(0),
        f(1).toLong,
        f(2),
        f(3).toLong,
        f(4),
        f(5),
        f(6).toLong, //点击品类ID
        f(7).toLong, //点击商品id
        f(8), //下单品类的ID集合
        f(9), //下单商品的ID集合
        f(10), //支付品类的ID集合
        f(11), //支付商品的ID集合
        f(12).toLong)
    })
   //flatMap出类别id的点击数、下单数、支付数量：(类别id, 0, 1, 0)
    val value1: RDD[totalcount] = value.flatMap(
      f => {
        if (f.click_category_id != -1) {
          List(totalcount(f.click_category_id.toString, 1, 0, 0))
        } else if (f.order_category_ids != null) {
          val totalcounts: ListBuffer[totalcount] = new ListBuffer[totalcount]
          for (elem <- f.order_category_ids.split(",")) {
            val totalcount1: totalcount = totalcount(elem, 0, 1, 0)
            totalcounts.append(totalcount1)  //返回值会展开
          }
          totalcounts
          //为什么在这里返回的不是list->函数使用的是flatMap

        } else if (f.pay_category_ids != null) {
          val totalcounts: ListBuffer[totalcount] = new ListBuffer[totalcount]
          for (elem <- f.pay_category_ids.split(",")) {
            val totalcount1: totalcount = totalcount(elem, 0, 0, 1)
            totalcounts.append(totalcount1)
          }
          totalcounts
        } else {
          Nil
        }

      }
    )
    //对类别分组
    val value2: RDD[(String, Iterable[totalcount])] = value1.groupBy(f => f.id)
    //根据点击次数排序取类的前十
    value2
      .mapValues(f => {
        f.reduce {
          (info1, info2) => {
            //对三元组求和
            info1.clinkcount = info1.clinkcount + info2.clinkcount
            info1.ordercount = info1.ordercount + info2.ordercount
            info1.paycount = info1.paycount + info2.paycount
            info1  //info1是三元组
          }
        }
      })
      .map(f=>f._2)
      //对三元组逐个排序
      .sortBy(f=>(f.clinkcount,f.ordercount,f.paycount),false)
      .take(10)
      .foreach(f=>println(f))
    sc.stop()
  }
}