[Spark] spark中API的使用

1. FileSystem

/** file system API */
import org.apache.hadoop.fs.{FileSystem, Path}
import org.apache.hadoop.conf.Configuration

val conf = new Configuration()
val path = new Path(dir)
val fs: FileSystem = FileSystem.get(path.toUri, conf)
if (fs.exists(path)) {
  colorful_green_println(path.toString + " exists and delete.")
  fs.delete(path)
}

2. CommendLine

import org.apache.commons.cli.{BasicParser, CommandLine, Options}
/** Parse the arguments */
val options: Options = new Options()
options.addOption("yesterday", true, "The date of yesterday.")
options.addOption("window", true, "The window of data we used.")
options.addOption("base_dir", true, "The base dir of path")

val parser: BasicParser = new BasicParser()
val cl: CommandLine = parser.parse(options, args)
val yesterday = cl.getOptionValue("yesterday")
val window = cl.getOptionValue("window").toInt
val base_dir = cl.getOptionValue("base_dir")