Spark 排序+去重

数据：

1
1
2
2
3
4
2
1
4

代码：

import org.apache.spark.{SparkConf, SparkContext}

object SortAndDistinct {

  System.setProperty("hadoop.home.dir","d://soft/hadoop/hadoop-2.9.2")
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setMaster("local[1]").setAppName("sort-and-distinct")
    val sc = new SparkContext(conf)

    sc.textFile("f://out/distinct.txt")
      .distinct()
      .sortBy(line => line.toInt)
      .coalesce(1)
      .foreach(println)

    sc.stop()
  }

}

结果：

1
2
3
4