spark 显示hdfs 路径_如何在Spark Scala shell中列出HDFS位置中的所有csv文件？

我没有彻底测试它,但这样的东西似乎工作：

import org.apache.spark.deploy.SparkHadoopUtil

import org.apache.hadoop.fs.{FileSystem, Path, LocatedFileStatus, RemoteIterator}

import java.net.URI

val path: String = ???

val hconf = SparkHadoopUtil.get.newConfiguration(sc.getConf)

val hdfs = FileSystem.get(hconf)

val iter = hdfs.listFiles(new Path(path), false)

def listFiles(iter: RemoteIterator[LocatedFileStatus]) = {

def go(iter: RemoteIterator[LocatedFileStatus], acc: List[URI]): List[URI] = {

if (iter.hasNext) {

val uri = iter.next.getPath.toUri

go(iter, uri :: acc)

} else {

acc

}

}

go(iter, List.empty[java.net.URI])

}

listFiles(iter).filter(_.toString.endsWith(".csv"))