spark 自定义partitioner

最新推荐文章于 2023-12-31 23:59:54 发布

passer199101

最新推荐文章于 2023-12-31 23:59:54 发布

阅读量719

点赞数

分类专栏： spark 大数据

本文链接：https://blog.csdn.net/u012307002/article/details/61202949

版权

大数据同时被 2 个专栏收录

44 篇文章 0 订阅

订阅专栏

spark

41 篇文章 0 订阅

订阅专栏

package inpv

import org.apache.spark.Partitioner

class PvPartitioner(numParts: Int) extends Partitioner {
    override def numPartitions: Int = numParts

    override def getPartition(key: Any): Int = {
        val code = (key.toString().hashCode % numPartitions)
        if (code < 0) {
            code + numPartitions
        } else {
            code
        }
    }

    override def equals(other: Any): Boolean = other match {
        case p: PvPartitioner =>
            p.numPartitions == numPartitions
        case _ =>
            false
    }

    override def hashCode: Int = numPartitions
}