Spark 练习题-自定义分区

题一

package com.gxxd.spark.core
import org.apache.spark.rdd.RDD
import org.apache.spark.{Partitioner, SparkConf, SparkContext}

object RDD_Park {
  def main(args: Array[String]): Unit = {
    val sparkConf = new SparkConf().setMaster("local[*]").setAppName("Part")
    val sc = new SparkContext(sparkConf)
    var rdd = sc.makeRDD(List(
      ("nba", "welcome to nba"),
      ("cba", "welcome to cba"),
      ("wcba", "welcome to wcba"),
      ("nba", "welcome to nba too"),
    ), 3)

    val partRDD: RDD[(String, String)] = rdd.partitionBy(new MyPartitioner)

    val newRDD: RDD[(Int, (String, String))] = partRDD.mapPartitionsWithIndex((index, iter) => {
      iter.map({
        num => (index, num)
      })
    })
    newRDD.collect().foreach(println)
  }
  class MyPartitioner extends Partitioner {
    //  分区数
    override def numPartitions: Int = 3
    // 根据数据的key所在分区返回分区索引
    override def getPartition(key: Any): Int = {
      key match {
        case "nba" => 0
        case "cba" => 1
        case _ => 2
      }
    }
  }
}

题二

import org.apache.spark.rdd.RDD
import org.apache.spark.{Partitioner, SparkConf, SparkContext}

import scala.util.matching.Regex

object Ten5 {
  def main(args: Array[String]): Unit = {
    val sparkConf = new SparkConf().setMaster("local[*]").setAppName("Ten5")
    val sc = new SparkContext(sparkConf)
    val fileRDD = sc.textFile("data/data.txt")
    val groupRDD = fileRDD.groupBy(word => word)
    val partRDD = groupRDD.partitionBy(new MyPartitioner)
    val newRDD: RDD[(Int, (String, Iterable[String]))] = partRDD.mapPartitionsWithIndex((index, iter) => {
      iter.map({
        num => (index, num)
      })
    })
    newRDD.collect().foreach(println)
  }
  /**
   * 自定义分区器
   * 1.继承Partitioner
   * 2.重写Partitioner抽象类的方法
   */
  class MyPartitioner extends Partitioner {
    //  分区数
    override def numPartitions: Int = 4
    // 根据数据的key所在分区返回分区索引
    override def getPartition(key: Any): Int = {
      val r: Regex = """([a-z])([a-z])([a-z])""".r
      val r1: Regex = """([A-Z])([A-Z])([A-Z])""".r
      val r2: Regex = """([1-9])([1-9])([1-9])""".r
      key match {
        case r(x, y, z) => 0
        case r1(x, y, z) => 1
        case r2(x, y, z) => 2
        case _ => 3
      }
    }
  }
}