Spark-Sql源码解析之八 Codegen

最新推荐文章于 2022-01-02 08:57:17 发布

亮亮-AC米兰

最新推荐文章于 2022-01-02 08:57:17 发布

阅读量6.1k

点赞数 1

本文链接：https://blog.csdn.net/wl044090432/article/details/52190736

版权

本文深入探讨了Spark SQL中Codegen的作用，对比了动态字节码技术开启和关闭时Sort操作的性能差异。关闭时，Compare函数包含虚函数调用和装箱操作，消耗更多时间。而开启动态字节码技术后，性能得到显著提升。

摘要由CSDN通过智能技术生成

Codegen，动态字节码技术，那么什么是动态字节码技术呢？先看来一段代码，假设SparkPlan为Sort

case class Sort(
    sortOrder: Seq[SortOrder],
    global: Boolean,
    child: SparkPlan)
  extends UnaryNode {
  override def requiredChildDistribution: Seq[Distribution] =
    if (global) OrderedDistribution(sortOrder) :: Nil else UnspecifiedDistribution :: Nil

  protected override def doExecute(): RDD[Row] = attachTree(this, "sort") {
    child.execute().mapPartitions( { iterator =>
      val ordering = newOrdering(sortOrder, child.output)
      iterator.map(_.copy()).toArray.sorted(ordering).iterator
    }, preservesPartitioning = true)
  }

  override def output: Seq[Attribute] = child.output

  override def outputOrdering: Seq[SortOrder] = sortOrder
}

abstract class SparkPlan extends QueryPlan[SparkPlan] with Logging with Serializable {
protected def newOrdering(order: Seq[SortOrder], inputSchema: Seq[Attribute]): Ordering[Row] = {
  if (codegenEnabled) {//开启动态字节码技术
    GenerateOrdering.generate(order, inputSchema)
  } else {//否则关闭
    new RowOrdering(order, inputSchema)