案例 计算店铺的月销售额和累加到当前月的销售和--spark程序编写

/**
 * 数据：
 * shop1,2019-01-18,500
 * shop1,2019-02-10,500
 * shop1,2019-02-10,200
 * shop1,2019-02-11,600
 * shop1,2019-02-12,400
 * shop1,2019-02-13,200
 * shop1,2019-02-15,100
 * shop2,2019-02-10,100
 * shop2,2019-02-11,100
 * shop2,2019-02-13,100
 * shop2,2019-03-15,100
 * shop2,2019-04-15,100
 *
 * 思路：先根据sid，月份聚合销售额
 *      再根据sid分组,日期排序，累加月销售额
 */
object RollUpMonthIncome {
  def main(args: Array[String]): Unit = {
    val sc = SparkUtils.createContext(true)
    val lines = sc.textFile("data/shop.txt")

    //根据sid，月份聚合
    val reduced: RDD[((String, String), Int)] = lines.map(e => {
      val fields = e.split(",")
      val sid = fields(0)
      val month = fields(1).substring(0, 7)
      val money = fields(2).toInt
      //将shopid和month作为key
      ((sid, month), money)
    }).reduceByKey(_ + _)


    //根据sid分组,日期排序，累加
    //(shop1,2019-01),500
    //(shop1,2019-02),2500
    val res: RDD[(String, String, Int, Int)] = reduced.groupBy(_._1._1).flatMapValues(it => {
      //根据月份排序
      val sorted: List[((String, String), Int)] = it.toList.sortBy(_._1._2)
      var rollUp = 0

      sorted.map(e => {
        val sid = e._1._1
        val month = e._1._2
        val money = e._2
        //累加月销售额
        rollUp += money
        (sid, month, money, rollUp)
      })

    }).map(e => (e._1, e._2._2, e._2._3, e._2._4))


//    res.foreach(println)
    println(res.collect().toBuffer)
    sc.stop()

  }

}

工具类

//本地运行模式和集群运行模
object SparkUtils {
  def createContext(isLocal: Boolean = false): SparkContext = {
    val conf = new SparkConf().setAppName(this.getClass.getSimpleName)
    if (isLocal) conf.setMaster("local[*]")
    val sc = new SparkContext(conf)
    sc
  }

}