spark读取多目录生成parquet文件

package mcd.etl.cosmos
import java.io.File
import org.apache.spark.SparkConf
import org.apache.spark.sql.SparkSession
object scalaMegreData {
def main(args: Array[String]): Unit = {
val conf = new SparkConf().setMaster(“local[*]”).setAppName(“scalaMegreData”)
val spark = SparkSession.builder().config(conf).getOrCreate()
val path: File = new File(“D:\DataTest\”)
for (d <- subdirs3(path)){
val aa = d.toString
spark.read.format(“csv”).option(“header”, “true”).load(aa).createOrReplaceTempView(“tmp”)
val dates = Array(“2010”,“2011”,“2012”,“2013”,“2014”,“2015”,“2016”,“2017”,“2018”,“2019”,“2020”)
val path1 = aa.split("\\")(2)
println(path1)
for(date <- dates){
val dataSql = spark.sql(s"select * From tmp where substr(date,1,4)=$date").repartition(1)
.write.mode(“Append”).parquet(“D:\data\datatest\” +path1+ s"\$date")
}
}
}

def subdirs3(dir: File): Iterator[File] = {
val d = dir.listFiles.filter(_.isDirectory)
val f = dir.listFiles.toIterator
f ++ d.toIterator.flatMap(subdirs3 _)}
}