scala读取本地hdfs_Scala操作HDFS

通过Scala对HDFS的一些操作，包括创建目录，删除目录，上传文件，文件读取，删除文件，Append文件等等；

import java.io._

import java.net.URI

import java.util._

import org.apache.commons.lang3.StringUtils

import org.apache.hadoop.conf.Configuration

import org.apache.hadoop.fs._

import org.apache.zookeeper.common.IOUtils

/**

* 通过scala操作HDFS

*

* @author Sunny

* @taskId:

* @version 1.0

* @createDate 2018/06/11 9:29

* @see com.spark.sunny.hdfs

*/

object HDFSUtil {

val hdfsUrl = "hdfs://iotsparkmaster:9000"

var realUrl = ""

/**

* make a new dir in the hdfs

*

* @param dir the dir may like '/tmp/testdir'

* @return boolean true-success, false-failed

*/

def mkdir(dir : String) : Boolean = {

var result = false

if (StringUtils.isNoneBlank(dir)) {

realUrl = hdfsUrl + dir

val config = new Configuration()

val fs = FileSystem.get(URI.create(realUrl), config)

if (!fs.exists(new Path(realUrl))) {

fs.mkdirs(new Path(realUrl))

}

fs.close()

result = true

}

result

}

/**

* delete a dir in the hdfs.

* if dir not exists, it will throw FileNotFoundException

*

* @param dir the dir may like '/tmp/testdir'

* @return boolean true-success, false-failed

*

*/

def deleteDir(dir : String) : Boolean = {

var result = false

if (StringUtils.isNoneBlank(dir)) {

realUrl = hdfsUrl + dir

val config = new Configuration()

val fs = FileSystem.get(URI.create(realUrl), config)

fs.delete(new Path(realUrl), true)

fs.close()

result = true

}

result

}

/**

* list files/directories/links names under a directory, not include embed

* objects

*

* @param dir a folder path may like '/tmp/testdir'

* @return List list of file names

*/

def listAll(dir : String) : List[String] = {