136.Spark大型电商项目-用户活跃度分析模块-统计最近一个周期相对上一个周期消费金额增长最多的10个用户

目录

代码

运行结果

---

本篇文章记录获取指定时间范围内消费金额增长最多的10个用户，使用SparkSession来分析数据。

代码

UserActiveDegreeAnalyze.scala 

package graduation.scala.spark

import org.apache.spark.SparkConf
import org.apache.spark.sql.SparkSession

/**
  * FileName: UserActiveDegreeAnalyze
  * Author:   hadoop
  * Email:    3165845957@qq.com
  * Date:     19-8-4 下午11:31
  * Description:
  *
  * 用户活跃度分析
  *
  * 我们这次项目课程的升级，也跟spark从入门到精通的升级采取同步，采用scala+eclipse的方式来开发
  *
  * 我个人而言，还是觉得应该用java去开发spark作业，因为hadoop是最重要的大数据引擎，hadoop mapreduce、hbase，全都是java
  * 整个公司的编程语言技术栈越简单越好，降低人员的招聘和培养的成本
  *
  * 但是由于市面上，现在大部分的公司，做spark都是采取一种，spark用scala开发，所以开发spark作业也用scala
  * 课程为了跟市场保持同步，后面就随便采取scala来开发了
  *
  */
object UserActiveDegreeAnalyze {

  case class UserActionLog(logId: Long, userId: Long, actionTime: String, actionType: Long, purchaseMoney: Double)
  case class UserActionLogVO(logId: Long, userId: Long, actionValue: Long)
  case class UserActionLogWithPurchaseMoneyVO(logId: Long, userId: Long, purchaseMoney: Double)

  def main(args: Array[String]) {
    // 如果是按照课程之前的模块，或者整套交互式分析系统的架构，应该先从mysql中提取用户指定的参数（java web系统提供界面供用户选择，然后java web系统将参数写入mysql中）
    // 但是这里已经讲了，之前的环境已经没有了，所以本次升级从简
    // 我们就直接定义一个日期范围，来模拟获取了参数
    val startDate = "2019-08-01";
    val endDate = "2019-09-01";

    // 开始写代码
    // spark 2.0具体开发的细节和讲解，全部在从入门到精通中，这里不多说了，直接写代码
    // 要不然如果没有看过从入门到精通的话，就自己去上网查spark 2.0的入门资料
    val conf  = new SparkConf().setAppName("UserActiveDegreeAnalyze").setMaster("local[2]")
    /*val spark = SparkSession
      .builder()
      .appName("UserActiveDegreeAnalyze")
      .master("local")
      .config("spark.sql.warehouse.dir", "D:\\test\\spark\\mall\\spark-warehouse")
      .getOrCreate()*/
    val  spark = SparkSession.builder().config(conf).getOrCreate()
    val sc = spark.sparkContext
    // 导入spark的隐式转换
    import spark.implicits._
    // 导入spark sql的functions
    import org.apache.spark.sql.functions._

    val  dataPath:String ="/home/hadoop/IdeaProjects/BigDataGraduationProject/log/"
    // 获取两份数据集
    val userBaseInfo = spark.read.json(dataPath+"user_base_info.json")
    val userActionLog = spark.read.json(dataPath+"user_action_log.json")

    //第四个功能：统计最近一个周期相对上一个周期消费金额增长最多的10个用户

    val userActionLogWithPurchaseMoneyInFirstPeriod = userActionLog.as[UserActionLog]
        .filter("actionTime >= '2019-10-01' and actionTime <= '2019-10-31' and actionType = 1")
        .map{ userActionLogEntry =>
          UserActionLogWithPurchaseMoneyVO(userActionLogEntry.logId, userActionLogEntry.userId,
            userActionLogEntry.purchaseMoney) }

    val userActionLogWithPurchaseMoneyInSecondPeriod = userActionLog.as[UserActionLog]
        .filter("actionTime >= '2019-09-01' and actionTime <= '2019-09-30' and actionType = 1")
        .map{ userActionLogEntry =>
          UserActionLogWithPurchaseMoneyVO(userActionLogEntry.logId, userActionLogEntry.userId,
            -userActionLogEntry.purchaseMoney) }

    val userActionLogWithPurchaseMoneyDS = userActionLogWithPurchaseMoneyInFirstPeriod.union(userActionLogWithPurchaseMoneyInSecondPeriod)

    userActionLogWithPurchaseMoneyDS
        .join(userBaseInfo, userActionLogWithPurchaseMoneyDS("userId") === userBaseInfo("userId"))
        .groupBy(userBaseInfo("userId"), userBaseInfo("username"))
        .agg(round(sum(userActionLogWithPurchaseMoneyDS("purchaseMoney")), 2).alias("purchaseMoneyIncr"))
        .sort($"purchaseMoneyIncr".desc)
        .limit(10)
        .show()

    sc.stop()
    spark.stop()

  }

}

运行结果

有时间再贴结果