spark中,检测hdfs文件夹是否存在 和 删除文件
import org.apache.hadoop.fs.{FileSystem, Path}
import org.apache.spark.ml.common.HdfsHelper.isDir
1、检测hdfs文件路径是否存在
def pathIsExist(spark: SparkSession, path: String): Boolean = {
//取文件系统
val filePath = new org.apache.hadoop.fs.Path( path )
val fileSystem = filePath.getFileSystem( spark.sparkContext.hadoopConfiguration )
// 判断路径是否存在
fileSystem.exists( filePath )
}
2、删除文件
def deleteFile(spark: SparkSession, path: String) : Boolean = {
val filePath = new org.apache.hadoop.fs.Path( path )
val fileSystem = filePath.getFileSystem( spark.sparkContext.hadoopConfiguration )
if (isDir(fileSystem, path))
fileSystem.delete(new Path(path), true)//true: delete files recursively
else
fileSystem.delete(new Path(path), false)
}