错误:Caused by: org.apache.spark.SparkException: A master URL must be set in your configuration
at org.apache.spark.SparkContext.<init>(SparkContext.scala:401)
原因:创建spark实例,或者sc.textFile读取数据等放在了main函数的外面。
val conf = new SparkConf().setAppName("appname")
val sc = new SparkContext(conf)
def main(args: Array[String]): Unit = {
if (args.length != 3) {
正确写法:
def main(args: Array[String]): Unit = {
if (args.length != 3) {
System.err.println(
s"""
|Usage: InputStreaming <startDate> <endDate> [格式:yyyyMMdd]
|<startDate> 开始日期
|<endDate> 结束日期
""".stripMargin)
}
val conf = new SparkConf().setAppName("appname")
val sc = new SparkContext(conf)