Java API 读写 hdfs 操作
一、概述
操作hdfs分布式文件系统的类为FileSystem对象
1、创建文件夹(递归创建):fs. mkdirs()
2、写文件 : fs.create(new Path()),返回FSDataOutputStream对象
3、读文件 : fs.open(new Path()) , 返回FSDataInputStream对象
二、代码
package com.example.hdfs;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import java.net.URI;
import java.util.logging.Logger;
public class TestMain {
private static final Logger logger = Logger.getLogger("com.example.hdfs.TestMain");
public static void main(String[] args) throws Exception {
//HDFS URI}
String hdfsuri = "hdfs://localhost:8020/";
String path="/user/hdfs/example/hdfs/";
String fileName="hello.csv";
String fileContent="hello;world";
// ====== Init HDFS File System Object
Configuration conf = new Configuration();
// Set FileSystem URI
conf.set("fs.defaultFS", hdfsuri);
// Because of Maven
conf.set("fs.hdfs.impl", org.apache.hadoop.hdfs.DistributedFileSystem.class.getName());
conf.set("fs.file.impl", org.apache.hadoop.fs.LocalFileSystem.class.getName());
// 设置hadoop应用程序执行用户
System.setProperty("HADOOP_USER_NAME", "hdfs");
System.setProperty("hadoop.home.dir", "/");
//创建 FileSystem - HDFS
FileSystem fs = FileSystem.get(URI.create(hdfsuri), conf);
//创建文件目录,如果不存在
Path workingDir=fs.getWorkingDirectory();
Path newFolderPath= new Path(path);
if(!fs.exists(newFolderPath)) {
// Create new Directory
fs.mkdirs(newFolderPath);
logger.info("Path "+path+" created.");
}
//==== Write file
logger.info("Begin Write file into hdfs");
//Create a path
Path hdfswritepath = new Path(newFolderPath + "/" + fileName);
//Init output stream
FSDataOutputStream outputStream=fs.create(hdfswritepath);
//Cassical output stream usage
outputStream.writeBytes(fileContent);
outputStream.close();
logger.info("End Write file into hdfs");
//==== Read file
logger.info("Read file into hdfs");
//Create a path
Path hdfsreadpath = new Path(newFolderPath + "/" + fileName);
//Init input stream
FSDataInputStream inputStream = fs.open(hdfsreadpath);
//Classical input stream usage
String out= IOUtils.toString(inputStream, "UTF-8");
logger.info(out);
inputStream.close();
fs.close();
}
}