HDFS JAVA API简单测试

创建目录

mkdirs相当于 mkdir -p, mkdirs在dir存在的情况下也是返回true

public static void mkdirs(FileSystem fs, String path) {
		try {
			Path dirPath = new Path(path);
			if (!fs.exists(dirPath)) {
				fs.mkdirs(dirPath);
				System.out.println("created dir:" + path);
			} else {
				System.out.println("dir:" + path + "already exists");
			}
		} catch (IOException e) {
			System.out.println("mkdir:" + path + " failed, msg="+ e.getMessage());
			e.printStackTrace();
		}
	}

创建新文件

文件的父目录会自动创建

	public static void createFile(FileSystem fs, String filePath, String content) {
		try {
			Path path = new Path(filePath);
			FSDataOutputStream out = fs.create(path);
			out.write(content.getBytes());
			System.out.println("create File:" + filePath + " successed");
			out.close();
		} catch (IOException e) {
			System.out.println("create File:" + filePath + " failed, msg="+ e.getMessage());
			e.printStackTrace();
		}
	}

文件重命名

to目录必须存在，否则rename会失败，文件不会移动到目的路径

	public static void mv(FileSystem fs, String from, String to) {
		try {
			fs.rename(new Path(from), new Path(to));
			System.out.println("mv from:" + from + " to " + to+ " successed");
		} catch (IOException e) {
			System.out.println("mv from:" + from + " to " + to+ " failed, msg="+ e.getMessage());
			e.printStackTrace();
		}
	}

删除文件或目录

	public static void delete(FileSystem fs, String filePath) {
		try {
			fs.delete(new Path(filePath), true);// true表示是否递归删除目录，如果是文件则无效
			System.out.println("delete file:" + filePath + " successed");
		} catch (IOException e) {
			System.out.println("delete file:" + filePath + " delete failed, msg="+ e.getMessage());
			e.printStackTrace();
		}
	}

上传本地文件

hdfsPath的目录会自动创建

public static void uploadFile(FileSystem fs, String localPath, String hdfsPath,boolean deleteLocal){
		if(!new File(localPath).exists()){
			System.out.println("localfile: "+localPath+" does not exist");
			return;
		}
		try {
			//deleteLocal为false表示不删除源文件,true为删除
			fs.copyFromLocalFile(deleteLocal, new Path(localPath), new Path(hdfsPath));
			System.out.println("uploadFile File:" + localPath + " to: "+hdfsPath+" successed");
		} catch (IOException e) {
			System.out.println("uploadFile File:" + localPath + " to: "+hdfsPath+" failed,msg="+e.getMessage());
			e.printStackTrace();
		}
	}

下载HDFS文件到本地

本地目录会自动创建

public static void downloadFiles(FileSystem fs, String hdfsPath, String localPath,boolean deleteSrc){
		try {
			if(!fs.exists(new Path(hdfsPath))){
				System.out.println("hdfsPath: "+hdfsPath+" does not exist");
				return;
			}
			fs.copyToLocalFile(deleteSrc, new Path(hdfsPath), new Path(localPath));
			System.out.println("downloadFiles File from:" + hdfsPath + " to: "+localPath+" successed");
		} catch (IOException e) {
			System.out.println("downloadFiles File from:" + hdfsPath + " to: "+localPath+" failed,msg="+e.getMessage());
			e.printStackTrace();
		}
	}

读取文件的内容

public static void printFile(FileSystem fs, String filePath) throws IOException{
		Path srcPath = new Path(filePath);
		InputStream in = null;
		try {
			in = fs.open(srcPath);
			IOUtils.copyBytes(in, System.out, 4096, false);
		} finally {
			IOUtils.closeStream(in);
		}

	}

读取目录信息

	public static void printStatus(FileSystem fs, String hdfsPath){
		try {
              FileStatus[] files = fs.listStatus(new Path(hdfsPath));
              for(FileStatus file: files)
            	  System.out.println(file.getPath().toString());
		} catch (IOException e) {
			System.out.println("printStatus :" + hdfsPath + " failed,msg="+e.getMessage());
			e.printStackTrace();
		}
		
	}

测试

	public static void main(String[] args) throws Exception {
		Configuration conf = new Configuration();
		conf.set("fs.defaultFS", "hdfs://localhost:9000");
		FileSystem fs = FileSystem.get(conf);
		mkdirs(fs, "/hadoop/test");
		createFile(fs, "/hadoop/test1/files/helloworld.txt", "helloscala");
		mkdirs(fs, "/hadoop/test3/files/");
		mv(fs, "/hadoop/test1/files/helloworld.txt","/hadoop/test3/files/hellohdfs.txt");
		delete(fs, "/helloworld");
		uploadFile(fs,"D:/hadoop-2.7.3/LICENSE.txt","/hadoop/files/license.txt",false);
		downloadFiles(fs,"/files","D:/hdfs/files",false);
		printStatus(fs,"/files");
		printFile(fs,"/hadoop/files/license.txt");
	}

转载于:https://my.oschina.net/Endless2010/blog/1504218