0
点赞
收藏
分享

微信扫一扫

spark 自定义partitioner

穆风1818 2023-04-14 阅读 111


package inpv

import org.apache.spark.Partitioner

class PvPartitioner(numParts: Int) extends Partitioner {
    override def numPartitions: Int = numParts

    override def getPartition(key: Any): Int = {
        val code = (key.toString().hashCode % numPartitions)
        if (code < 0) {
            code + numPartitions
        } else {
            code
        }
    }

    override def equals(other: Any): Boolean = other match {
        case p: PvPartitioner =>
            p.numPartitions == numPartitions
        case _ =>
            false
    }

    override def hashCode: Int = numPartitions
}


举报

相关推荐

0 条评论