java查看hdfs文件属性_Java查询HDFS文件系统

1、文件元数据 FileStatus

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.FileStatus;

import org.apache.hadoop.fs.FileSystem;

import org.apache.hadoop.fs.FsStatus;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.fs.permission.FsPermission;

import java.io.IOException;

import java.net.URI;

public class ShowFileStatus {

public static void main(String[] args) {

try {

URI uri = URI.create("hdfs://192.168.1.100:9000/home/ossuser/1.txt");

Configuration conf = new Configuration();

FileSystem fs = FileSystem.get(uri, conf);

Path path = new Path(uri);

FsStatus status = fs.getStatus(path);

System.out.println("fsStatus.getCapacity() = " + status.getCapacity());

System.out.println("fsStatus.getUsed() = " + status.getUsed());

System.out.println("fsStatus.getRemaining() = " + status.getRemaining());

System.out.println("----------------------------------");

FileStatus fileStatus = fs.getFileStatus(path);

System.out.println("fileStatus.getOwner() = " + fileStatus.getOwner());

System.out.println("fileStatus.getGroup() = " + fileStatus.getGroup());

System.out.println("fileStatus.getAccessTime() = " + fileStatus.getAccessTime());

System.out.println("fileStatus.getBlockSize() = " + fileStatus.getBlockSize());

System.out.println("fileStatus.getLen() = " + fileStatus.getLen());

System.out.println("fileStatus.getModificationTime() = " + fileStatus.getModificationTime());

System.out.println("fileStatus.getReplication() = " + fileStatus.getReplication());

FsPermission fsPermission = fileStatus.getPermission();

System.out.println("fileStatus.getPermission() = " + fsPermission);

} catch (IOException e) {

e.printStackTrace();

}

}

}

输入如下：

fsStatus.getCapacity() = 38046990336

fsStatus.getUsed() = 187760640

fsStatus.getRemaining() = 14045487104

----------------------------------

fileStatus.getOwner() = ossuser

fileStatus.getGroup() = supergroup

fileStatus.getAccessTime() = 1555746057611

fileStatus.getBlockSize() = 134217728

fileStatus.getLen() = 1935838

fileStatus.getModificationTime() = 1555746058582

fileStatus.getReplication() = 3

fileStatus.getPermission() = rw-r--r--

FileStatus封装了目录与文件的元数据信息，包括所有者、块大小、文件长度、修改时间、副本数、权限等信息

2、列出文件

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.FileStatus;

import org.apache.hadoop.fs.FileSystem;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.fs.PathFilter;

import java.io.IOException;

import java.net.URI;

public class ShowFileStatus {

public static void main(String[] args) {

try {

URI uri = URI.create("hdfs://192.168.1.100:9000/home/ossuser/");

Configuration conf = new Configuration();

FileSystem fs = FileSystem.get(uri, conf);

Path path = new Path(uri);

FileStatus[] fileStatuses = fs.listStatus(path, new PathFilter() {

@Override

public boolean accept(Path path) {

try {

return fs.getFileStatus(path).isDirectory();

} catch (IOException e) {

return false;

}

}

});

for (FileStatus fileStatus : fileStatuses) {

System.out.println(fileStatus.getPath());

}

} catch (IOException e) {

e.printStackTrace();

}

}

}

输出结果

hdfs://192.168.1.100:9000/home/ossuser/aa

输出多个路径的文件列表信息：

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.FileStatus;

import org.apache.hadoop.fs.FileSystem;

import org.apache.hadoop.fs.FileUtil;

import org.apache.hadoop.fs.Path;

import java.io.IOException;

import java.net.URI;

public class FileStat2Paths {

public static void main(String[] args) {

try {

Path[] paths = new Path[]{

new Path("hdfs://192.168.1.100:9000/"),

new Path("hdfs://192.168.1.100:9000/home/ossuser"),

};

Configuration configuration = new Configuration();

FileSystem fs = FileSystem.get(URI.create(paths[0].toString()), configuration);

FileStatus[] status = fs.listStatus(paths);

Path[] listedPaths = FileUtil.stat2Paths(status);

for(Path listedPath : listedPaths) {

System.out.println(listedPath);

}

} catch (IOException e) {

e.printStackTrace();

}

}

}

hdfs://192.168.1.100:9000/2019

hdfs://192.168.1.100:9000/home

hdfs://192.168.1.100:9000/home/ossuser/1.txt

hdfs://192.168.1.100:9000/home/ossuser/aa

hdfs://192.168.1.100:9000/home/ossuser/log1.txt

3、文件模式

在一个表达式中使用通配符(globbing)匹配多个文件，FileSystem为匹配通配符提供了两个方法：

public FileStatus[] globStatus(Path pathPattern) throws IOException ;

public FileStatus[] globStatus(Path pathPattern, PathFilter filter) throws IOException

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.FileStatus;

import org.apache.hadoop.fs.FileSystem;

import org.apache.hadoop.fs.Path;

import java.io.IOException;

import java.net.URI;

public class GlobStatus {

public static void main(String[] args) {

URI uri = URI.create("hdfs://192.168.1.100:9000/");

try {

Configuration configuration = new Configuration();

FileSystem fs = FileSystem.get(uri, configuration);

FileStatus[] status = fs.globStatus(new Path("/*/{01,02, ossuser}"));

for (FileStatus fileStatus : status) {

System.out.println(fileStatus.getPath());

}

} catch (IOException e) {

e.printStackTrace();

}

}

}

输出如下：

hdfs://192.168.1.100:9000/2019/01

hdfs://192.168.1.100:9000/2019/02

通配符只能作用于文件路径与名称，不能作用于文件属性，可结合PathFilter接口实现类以达成更加灵活的文件或目录过滤。