CombineByKey

package com.ws.spark

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

object CombineByKeyTest {


  def main(args: Array[String]): Unit = {

    val dataList: List[(String, String, Double)] = List(
      ("大宝", "数学", 100.0),
      ("大宝", "英语", 99.0),
      ("大宝", "英语", 99.0),
      ("念念", "语文", 66.0),
      ("兔兔", "生物", 99.0),
      ("念念", "地理", 99.0),
      ("rabbit", "语文", 88.0)
    )

    val conf = new SparkConf().setMaster("local[*]").setAppName("CombineByKeyTest")

    val sc = new SparkContext(conf)

    val parRDD: RDD[(String, String, Double)] = sc.parallelize(dataList)

    val nameAndScore: RDD[(String, Score)] = parRDD.map(x => {
      (x._1, Score(x._1, x._2, x._3))
    })

    val keyAndScore: RDD[(String, (Double, Int))] = nameAndScore.combineByKey(
      //若该key不存在,则创建一个combine,存在则直接合并值,即跳过第一个函数直接计算第二个函数mergeValue
      //传入的参数是RDD的value值
      (s: Score) => (s.score, 1),
      //将相同key的value值进行计算.此处的v的类型就是第一个函数combine结果的类型(s.score, 1)
      (v: (Double, Int), s: Score) => (v._1 + s.score, v._2 + 1),
      //如果key存在多个分区,将各个分区的createCombiner合并
      (v: (Double, Int), v2: (Double, Int)) => (v._1 + v2._1, v._2 + v2._2)
    )

    //计算平均分数,保留小数点2位
    val avgScore: RDD[(String, String)] = keyAndScore.map(x => {
      (x._1, (x._2._1 / x._2._2).formatted("%.2f"))
    })

    println(avgScore.collect().toBuffer)

    sc.stop()

  }


}

case class Score(name: String, subjet: String, score: Double)

结果 :

ArrayBuffer((兔兔,99.00), (念念,82.50), (rabbit,88.00), (大宝,99.33))