题一
package com.gxxd.spark.core
import org.apache.spark.rdd.RDD
import org.apache.spark.{Partitioner, SparkConf, SparkContext}
object RDD_Park {
def main(args: Array[String]): Unit = {
val sparkConf = new SparkConf().setMaster("local[*]").setAppName("Part")
val sc = new SparkContext(sparkConf)
var rdd = sc.makeRDD(List(
("nba", "welcome to nba"),
("cba", "welcome to cba"),
("wcba", "welcome to wcba"),
("nba", "welcome to nba too"),
), 3)
val partRDD: RDD[(String, String)] = rdd.partitionBy(new MyPartitioner)
val newRDD: RDD[(Int, (String, String))] = partRDD.mapPartitionsWithIndex((index, iter) => {
iter.map({
num => (index, num)
})
})
newRDD.collect().foreach(println)
}
class MyPartitioner extends Partitioner {
// 分区数
override def numPartitions: Int = 3
// 根据数据的key所在分区返回分区索引
override def getPartition(key: Any): Int = {
key match {
case "nba" => 0
case "cba" => 1
case _ => 2
}
}
}
}
题二
import org.apache.spark.rdd.RDD
import org.apache.spark.{Partitioner, SparkConf, SparkContext}
import scala.util.matching.Regex
object Ten5 {
def main(args: Array[String]): Unit = {
val sparkConf = new SparkConf().setMaster("local[*]").setAppName("Ten5")
val sc = new SparkContext(sparkConf)
val fileRDD = sc.textFile("data/data.txt")
val groupRDD = fileRDD.groupBy(word => word)
val partRDD = groupRDD.partitionBy(new MyPartitioner)
val newRDD: RDD[(Int, (String, Iterable[String]))] = partRDD.mapPartitionsWithIndex((index, iter) => {
iter.map({
num => (index, num)
})
})
newRDD.collect().foreach(println)
}
/**
* 自定义分区器
* 1.继承Partitioner
* 2.重写Partitioner抽象类的方法
*/
class MyPartitioner extends Partitioner {
// 分区数
override def numPartitions: Int = 4
// 根据数据的key所在分区返回分区索引
override def getPartition(key: Any): Int = {
val r: Regex = """([a-z])([a-z])([a-z])""".r
val r1: Regex = """([A-Z])([A-Z])([A-Z])""".r
val r2: Regex = """([1-9])([1-9])([1-9])""".r
key match {
case r(x, y, z) => 0
case r1(x, y, z) => 1
case r2(x, y, z) => 2
case _ => 3
}
}
}
}