Spark二次排序简单实现

最新推荐文章于 2022-07-22 15:37:35 发布

yj2434

最新推荐文章于 2022-07-22 15:37:35 发布

阅读量161

点赞数

分类专栏： spark

本文链接：https://blog.csdn.net/yj2434/article/details/109348271

版权

spark 专栏收录该内容

16 篇文章 0 订阅

订阅专栏

import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.rdd.RDD

case class SecondSortKey(first: Int, second: Int) extends Ordered[SecondSortKey] {
  override def compare(that: SecondSortKey): Int = {
    if (this.first == that.first) {
      this.second - that.second
    } else {
      this.first - that.first
    }
  }
}

object SecondSort {
  def main(args: Array[String]): Unit = {
    val context: SparkContext = new SparkContext(new SparkConf()
      .setAppName("second sort")
      .setMaster("local"))
    val lines: RDD[String] = context.textFile("T:/code/spark_scala/data/secondSort.txt")

    lines.map(line => {
      (SecondSortKey(
        line.split(" ")(0).toInt,
        line.split(" ")(1).toInt), line)
    }).sortByKey()
      .map(_._2)
      .foreach(println)
  }
}