1.传参
spark-submit传递参数有两种方式:
- –conf k1=v1 --conf k2=v2
- cli args,在jar包后追加
详见官方文档:
2.解析
–conf方式解析:
sparkContext.getConf.get("k1")
cli args方式解析:
parse(args.toList)
... ...
def parse(list: List[String]): Unit = list match {
case "--input" :: value :: tail =>
input = value
parse(tail)
case "--output" :: value :: tail =>
output = value
parse(tail)
case "--tmpOutputDir" :: value :: tail =>
tmpOutputDir = value
parse(tail)
case "--sql" :: value :: tail =>
sql = URLDecoder.decode(value)
parse(tail)
case _ :: tail =>
parse(tail)
case Nil =>
}