spark -- groupbykey

需求：一组数按类别分组统计

spark 实现

import org.apache.spark.sql.{Row, SparkSession}

object groupbykey {

  def main(args: Array[String]): Unit = {
    val spark = SparkSession.builder().appName("----").master("local[*]").getOrCreate()
    val sc = spark.sparkContext
    val data = sc.parallelize(Seq(("C", Row(3)), ("A", Row(1)), ("B", Row(4)), ("A", Row(2)), ("B", Row(5))))
    val groupfunc = data.groupByKey()
    groupfunc.collect().foreach(println)
    import spark.implicits._
    val aa = groupfunc.map(x => {
      val key = x._1
      val values = x._2.toList
      var sum = 0
      for (value <- values) {
        sum = sum + value.getInt(0)
      }
      (key, Row(sum))
    })
    aa.collect().foreach(println)
    spark.close()
  }
}

---

结果