1. 执行如下SQL命令
sc.textFile("file:///D:\\myprojects\\SparkSQLPerf\\data\\students.txt")
.map(x => x.split(" "))
.map(x => (Student3(x(0), x(1), x(3), x(2).toInt))).toDF().registerTempTable("TBL_STUDENT")
val df = sqlContext.sql("describe TBL_STUDENT")
df.show()
2.执行结果
+--------+---------+-------+
|col_name|data_type|comment|
+--------+---------+-------+
| id| string| |
| name| string| |
| classId| string| |
| age| int| |
+--------+---------+-------+
3.物理计划执行结果
== Parsed Logical Plan ==
nodeName: <DescribeCommand>, argString:< ('nodeName: <UnresolvedRelation>, argString:< [TBL_STUDENT], None>), false>
== Analyzed Logical Plan ==
col_name: string, data_type: string, comment: string
nodeName: <DescribeCommand>, argString:< ('nodeName: <UnresolvedRelation>, argString:< [TBL_STUDENT], None>), false>
== Optimized Logical Plan ==
nodeName: <DescribeCommand>, argString:< ('nodeName: <UnresolvedRelation>, argString:< [TBL_STUDENT], None>), false>
== Not Prepared Physical Plan ==
nodeName: <ExecutedCommand>, argString:< nodeName: <DescribeCommand>, argString:< (Scan PhysicalRDD[AttributeReference:id#0,AttributeReference:name#1,AttributeReference:classId#2,AttributeReference:age#3]), [AttributeReference:col_name#6,AttributeReference:data_type#7,AttributeReference:comment#8], false>
>
== Prepared Physical Plan ==
nodeName: <ExecutedCommand>, argString:< nodeName: <DescribeCommand>, argString:< (Scan PhysicalRDD[AttributeReference:id#0,AttributeReference:name#1,AttributeReference:classId#2,AttributeReference:age#3]), [AttributeReference:col_name#6,AttributeReference:data_type#7,AttributeReference:comment#8], false>
>
4.DescribeCommand的代码
case class DescribeCommand(
child: SparkPlan,
override val output: Seq[Attribute],
isExtended: Boolean)
extends RunnableCommand {
override def run(sqlContext: SQLContext): Seq[Row] = {
val a = child.schema
val b = a.fields
b .map { field =>
val cmtKey = "comment"
val m = field.metadata
val comment = if (m.contains(cmtKey)) m.getString(cmtKey) else ""
//列名,列的类型,列的注释
Row(field.name, field.dataType.simpleString, comment)
}
}
}