HDFS的java操作方式
(1)使用协议方式访问hdfs上的文件(模仿http)
public class App1 {
// 访问的文件路径
static final String PATH = "hdfs://chaoren:9000/hello";
public static void main(String[] args) throws Exception {
//设置访问流的协议类型,指定为hdfs
URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory());
URL url = new URL(PATH);
InputStream in = url.openStream();
IOUtils.copyBytes(in, System.out, 1024, true);}
}
(2)使用FileSystem类对hdfs进行全方位操作(常用)
public class App2 {
static final String PATH = "hdfs://chaoren:9000/";
static final String DIR = "/dl";
static final String FILE = "/dl/hello";
public static void main(String[] args) throws Exception {
FileSystem fileSystem = getFileSystem();
// 创建文件夹 hadoop fs -mkdir /dl
//madirs(fileSystem);
//上传文件 hadoop fs -put src des
//putData(fileSystem);
//下载文件 hadoop fs -get src des
//getData(fileSystem);
//浏览文件夹
list(fileSystem);
//删除文件(夹)
//fileSystem.delete(new Path(DIR), true);
}
//浏览文件夹
private static void list(FileSystem fileSystem) throws IOException {
FileStatus[] listStatus = fileSystem.listStatus(new Path("/"));
for (FileStatus fileStatus : listStatus) {
String isDir= fileStatus.isDir()? "文件夹": "文件";
String permission = fileStatus.getPermission().toString();
short replication = fileStatus.getReplication();
long len = fileStatus.getLen();
String path = fileStatus.getPath().toString();
System.out.println(isDir+"\t"+permission+"\t"+replication+"\t"+len+"\t"+path);
}
}
@SuppressWarnings("unused")
//下载文件
hadoop
fs
-get
src
des
private static void getData(FileSystem fileSystem) throws IOException {
FSDataInputStream in = fileSystem.open(new Path(FILE));
IOUtils.copyBytes(in, System.out, 1024, true);
}
@SuppressWarnings("unused")
//上传文件
hadoop
fs
-put
src
des
private static void putData(FileSystem fileSystem) throws IOException,
FileNotFoundException {
FSDataOutputStream out = fileSystem.create(new Path(FILE));
FileInputStream in = new FileInputStream("G:/haha.txt");
IOUtils.copyBytes(in, out, 1024,true);
}
@SuppressWarnings("unused")
// 创建文件夹
hadoop
fs
-
mkdir
/
dl
private static void madirs(FileSystem fileSystem) throws IOException {
fileSystem.mkdirs(new Path(DIR));
}
private static FileSystem getFileSystem() throws Exception {
return FileSystem.get(new URI(PATH), new Configuration());
}
}