Flink源码-9-Flink sql翻译Job及优化

翻译过程

参考文档

示例SQL

SELECT
	*
FROM
	(
		(
			SELECT
				*
			FROM
				OrderA
			WHERE
				user < 3
		)
		UNION ALL
		(
			SELECT
				*
			FROM
				OrderB
			WHERE
				product <> 'rubber'
		)
	) OrderAll
WHERE
	amount > 2

查看执行计划和优化

这里可以看到明显使用了谓词下推
提前过滤掉amount>2

== Abstract Syntax Tree ==
LogicalProject(user=[$0], product=[$1], amount=[$2])
  LogicalFilter(condition=[>($2, 2)])
    LogicalUnion(all=[true])
      LogicalProject(user=[$0], product=[$1], amount=[$2])
        LogicalFilter(condition=[<($0, 3)])
          FlinkLogicalDataStreamScan(id=[1], fields=[user, product, amount])
      LogicalProject(user=[$0], product=[$1], amount=[$2])
        LogicalFilter(condition=[<>($1, _UTF-16LE'rubber')])
          FlinkLogicalDataStreamScan(id=[2], fields=[user, product, amount])

== Optimized Logical Plan ==
DataStreamUnion(all=[true], union all=[user, product, amount])
  DataStreamCalc(select=[user, product, amount], where=[AND(<(user, 3), >(amount, 2))])
    DataStreamScan(id=[1], fields=[user, product, amount])
  DataStreamCalc(select=[user, product, amount], where=[AND(<>(product, _UTF-16LE'rubber':VARCHAR(65536) CHARACTER SET "UTF-16LE"), >(amount, 2))])
    DataStreamScan(id=[2], fields=[user, product, amount])

== Physical Execution Plan ==
Stage 1 : Data Source
	content : collect elements with CollectionInputFormat

Stage 2 : Data Source
	content : collect elements with CollectionInputFormat

	Stage 3 : Operator
		content : from: (user, product, amount)
		ship_strategy : FORWARD

		Stage 4 : Operator
			content : where: (AND(<(user, 3), >(amount, 2))), select: (user, product, amount)
			ship_strategy : FORWARD

			Stage 5 : Operator
				content : from: (user, product, amount)
				ship_strategy : FORWARD

				Stage 6 : Operator
					content : where: (AND(<>(product, _UTF-16LE'rubber':VARCHAR(65536) CHARACTER SET "UTF-16LE"), >(amount, 2))), select: (user, product, amount)
					ship_strategy : FORWARD

源码

 override def explain(operations: util.List[Operation], extraDetails: ExplainDetail*): String = {
    require(operations.nonEmpty, "operations should not be empty")
    val sinkRelNodes = operations.map {
      case queryOperation: QueryOperation =>
        val relNode = getRelBuilder.queryOperation(queryOperation).build()
        relNode match {
          // SQL: explain plan for insert into xx
          case modify: LogicalTableModify =>
            // convert LogicalTableModify to CatalogSinkModifyOperation
            val qualifiedName = modify.getTable.getQualifiedName
            require(qualifiedName.size() == 3, "the length of qualified name should be 3.")
            val modifyOperation = new CatalogSinkModifyOperation(
              ObjectIdentifier.of(qualifiedName.get(0), qualifiedName.get(1), qualifiedName.get(2)),
              new PlannerQueryOperation(modify.getInput)
            )
            translateToRel(modifyOperation)
          case _ =>
            relNode
        }
      case modifyOperation: ModifyOperation =>
        translateToRel(modifyOperation)
      case o => throw new TableException(s"Unsupported operation: ${o.getClass.getCanonicalName}")
    }
   // 优化
    val optimizedRelNodes = optimize(sinkRelNodes)
    //转成执行树
    val execNodes = translateToExecNodePlan(optimizedRelNodes)

    //转成transformation
    val transformations = translateToPlan(execNodes)
    //由transformation 转成作业图
    val streamGraph = 
    ExecutorUtils.generateStreamGraph(getExecEnv, transformations)

    val sb = new StringBuilder
    sb.append("== Abstract Syntax Tree ==")
    sb.append(System.lineSeparator)
    sinkRelNodes.foreach { sink =>
      sb.append(FlinkRelOptUtil.toString(sink))
      sb.append(System.lineSeparator)
    }

    sb.append("== Optimized Physical Plan ==")
    sb.append(System.lineSeparator)
    val explainLevel = if (extraDetails.contains(ExplainDetail.ESTIMATED_COST)) {
      SqlExplainLevel.ALL_ATTRIBUTES
    } else {
      SqlExplainLevel.DIGEST_ATTRIBUTES
    }
    val withChangelogTraits = extraDetails.contains(ExplainDetail.CHANGELOG_MODE)
    optimizedRelNodes.foreach { rel =>
      sb.append(FlinkRelOptUtil.toString(
        rel,
        explainLevel,
        withChangelogTraits = withChangelogTraits))
      sb.append(System.lineSeparator)
    }

    sb.append("== Optimized Execution Plan ==")
    sb.append(System.lineSeparator)
    sb.append(ExecNodePlanDumper.dagToString(execNodes))

    if (extraDetails.contains(ExplainDetail.JSON_EXECUTION_PLAN)) {
      sb.append(System.lineSeparator)
      sb.append("== Physical Execution Plan ==")
      sb.append(System.lineSeparator)
      sb.append(streamGraph.getStreamingPlanAsJSON)
    }

    sb.toString()
  }