spark定制之二：start.scala中的DSL

最新推荐文章于 2024-03-17 18:17:27 发布

hyalone2010

最新推荐文章于 2024-03-17 18:17:27 发布

阅读量1.5k

点赞数

分类专栏： spark 文章标签： spark dsl 定制

本文链接：https://blog.csdn.net/hyalone2010/article/details/37566825

版权

spark 专栏收录该内容

8 篇文章 0 订阅

订阅专栏

通过类的隐式转换可以让写脚本变得更容易（写入"select * from testperson" hqlgo直接处结果，"select * from testperson" saveto somefile直接写入文件）：

def getRegisterString(rddname:String,classname:String,tablename:String,tabledef:String) : String = {
    val members = tabledef.split(",").map(_.trim.split(" ").filter(""!=)).map(x => (x(0).trim,x(1).trim.head.toString.toUpperCase+x(1).trim.tail))
    val classmemberdef = members.map(x => (x._1+":"+x._2)).mkString(",")
    val convertstr = members.map(x => x._2).zipWithIndex.map(x => "t("+x._2+").to"+x._1).mkString(",")
    return s"""
        case class ${classname}(${classmemberdef})
        val schemardd = ${rddname}.map(_.split("${FIELD_SEPERATOR}")).map(t=>${classname}(${convertstr}))
        hive.registerRDDAsTable(schemardd,"${tablename}")
    """
}

org.apache.spark.repl.Main.interp.command("""
class CommandTranslator(cmd:String) extends java.io.Serializable {

    def hqlgo()(implicit f: SchemaRDD => MySchemaRDD) = {
        lastrdd = hql(cmd)
        lastrdd.go()
    }

    def hqlsaveto(output: String)(implicit f: SchemaRDD => MySchemaRDD) = {
        lastrdd = hql(cmd)
        lastrdd.saveto(output)
    }

    def defineas(tabledef:String) = {
        if( tabledef != "" ) {
            org.apache.spark.repl.Main.interp.command(
                getRegisterString(cmd,cmd.toUpperCase,cmd,tabledef)
            )
        } else {
            org.apache.spark.repl.Main.interp.command(
                "hive.registerRDDAsTable(${cmd},\"${cmd}\")"
            )
        }
    }

    def from(filepath:String) {
        if( cmd.startsWith("create table ") ) {
            val tablename = cmd.substring(13).trim().split(" ")(0)
            val leftstr = cmd.substring(13).trim().substring(tablename.length).trim()
            val tabledef = leftstr.substring(1,leftstr.length-1).trim()
            val realfile = AutoFileUtil.regularFile(filepath)
            org.apache.spark.repl.Main.interp.command(
                "val "+tablename+" = sc.textFile(\""+realfile+"\")"
            )
            new CommandTranslator(tablename).defineas(tabledef)
        } else {
            println("usage:")
            println("\"create table sometablename (field1 string,field2 int...)\" from \"somefile or hdfs:somepath\"")
        }
    }
}
object CommandTranslator {
    implicit def stringToTranslator(cmd:String) = new CommandTranslator(cmd)

    def show(tabledata:Array[org.apache.spark.sql.Row]) = {
        tabledata.foreach( x => println(x.mkString("\t")))
    }
}
""")

def auto = CommandTranslator
import CommandTranslator._


def help = {
    println("""example:
        "select * from testperson" hqlgo
        "select * from testperson" hqlsaveto "hdfs://somedir"
        "select * from testperson" hqlsaveto "somelocalfile"
        "create table sometable (name string,age int,weight double)" from "hdfs:/test/testperson"
        auto show hqlresult
        "somerdddata" defineas "(name string,age int)"
        if you want to see the help of enveronment, please type :help
        """)
}