spark 递归删除目录的方法,会尝试两种做法
若第一种deleteRecursivelyUsingUnixNative不成功,会立即尝试第二种
// org.apache.spark.network.util.JavaUtils.java
/**
* Delete a file or directory and its contents recursively.
* Don't follow directories if they are symlinks.
*
* @param file Input file / dir to be deleted
* @throws IOException if deletion is unsuccessful
*/
public static void deleteRecursively(File file) throws IOException {
if (file == null) { return; }
// On Unix systems, use operating system command to run faster
// If that does not work out, fallback to the Java IO way
if (SystemUtils.IS_OS_UNIX) {
try {
deleteRecursivelyUsingUnixNative(file);
return;
} catch (IOException e) {
logger.warn("Attempt to delete using native Unix OS command failed for path = {}. " +
"Falling back to Java IO way", file.getAbsolutePath(), e);
}
}
deleteRecursivelyUsingJavaIO(file);
}
deleteRecursivelyUsingUnixNative
使用linux 命令删除
// org.apache.spark.network.util.JavaUtils.java
private static void deleteRecursivelyUsingUnixNative(File file) throws IOException {
ProcessBuilder builder = new ProcessBuilder("rm", "-rf", file.getAbsolutePath());
Process process = null;
int exitCode = -1;
try {
// In order to avoid deadlocks, consume the stdout (and stderr) of the process
builder.redirectErrorStream(true);
builder.redirectOutput(new File("/dev/null"));
process = builder.start();
exitCode = process.waitFor();
} catch (Exception e) {
throw new IOException("Failed to delete: " + file.getAbsolutePath(), e);
} finally {
if (process != null) {
process.destroy();
}
}
if (exitCode != 0 || file.exists()) {
throw new IOException("Failed to delete: " + file.getAbsolutePath());
}
}
deleteRecursivelyUsingJavaIO
private static void deleteRecursivelyUsingJavaIO(File file) throws IOException {
if (file.isDirectory() && !isSymlink(file)) {
IOException savedIOException = null;
for (File child : listFilesSafely(file)) {
try {
deleteRecursively(child);
} catch (IOException e) {
// In case of multiple exceptions, only last one will be thrown
savedIOException = e;
}
}
if (savedIOException != null) {
throw savedIOException;
}
}
boolean deleted = file.delete();
// Delete can also fail if the file simply did not exist.
if (!deleted && file.exists()) {
throw new IOException("Failed to delete: " + file.getAbsolutePath());
}
}