使用Java API操作HDFS文件系统

使用Java API操作HDFS文件系统

关键点：

• 1）创建Configuration
• 2）获取FileSystem
• 3）…就是你的HDFS API的操作

首先创建FileSystem

    构造一个访问指定HDFS系统的客户端对象
    第一个参数：HDFS的URI
    第二个参数：客户端指定的配置参数
    第三个参数：客户端的身份，说白了就是用户名

FileSystemfileSystem = FileSystem.get(new URI(“hdfs://hadoop000:8020”), new configuration, “hadoop”);

1.创建HDFS文件夹

@Test
    public void mkdir() throws Exception {
        fileSystem.mkdirs(new Path("/hdfsapi/test"));
    }

2.查看HDFS内容

import org.apache.hadoop.io.IOUtils;

@Test
    public void text()throws Exception {
        FSDataInputStream in = fileSystem.open(new Path("/cdh_version.properties"));
        IOUtils.copyBytes(in, System.out, 1024);
    }

3.创建文件

 public void create()throws Exception {
	        FSDataOutputStream out = fileSystem.create(new Path("/hdfsapi/test/b.txt"));
	        out.writeUTF("hello DM replication 1");
	        out.flush();
	        out.close();
    }

4.测试文件名更改

 @Test
    public void rename() throws Exception {
        Path oldPath = new Path("/hdfsapi/test/b.txt");
        Path newPath = new Path("/hdfsapi/test/c.txt");
        boolean result = fileSystem.rename(oldPath, newPath);
        System.out.println(result);

    }

5.拷贝本地文件到HDFS文件系统(put)

@Test
    public void copyFromLocalFile() throws Exception {
        Path src = new Path("/Users/rocky/data/hello.txt");
        Path dst = new Path("/hdfsapi/test/");
        fileSystem.copyFromLocalFile(src,dst);
    }

6.拷贝HDFS文件到本地(get)

@Test
    public void copyToLocalFile() throws Exception {
        Path src = new Path("/hdfsapi/test/hello.txt");
        Path dst = new Path("/Users/rocky/tmp/software");
        fileSystem.copyToLocalFile(src, dst);
    }

7.查看目标文件夹下的所有文件

@Test
    public void listFiles() throws Exception {
        FileStatus[] statuses = fileSystem.listStatus(new Path("/hdfsapi/test"));

        for(FileStatus file : statuses) {
            String isDir = file.isDirectory() ? "文件夹" : "文件";
            String permission = file.getPermission().toString();
            short replication = file.getReplication();
            long length = file.getLen();
            String path = file.getPath().toString();


            System.out.println(isDir + "\t" + permission
                    + "\t" + replication + "\t" + length
                    + "\t" + path
            );
        }

    }

8.递归查看目标文件夹下的所有文件

@Test
    public void listFilesRecursive() throws Exception {

        RemoteIterator<LocatedFileStatus> files = fileSystem.listFiles(new Path("/hdfsapi/test"), true);

        while (files.hasNext()) {
            LocatedFileStatus file = files.next();
            String isDir = file.isDirectory() ? "文件夹" : "文件";
            String permission = file.getPermission().toString();
            short replication = file.getReplication();
            long length = file.getLen();
            String path = file.getPath().toString();


            System.out.println(isDir + "\t" + permission
                    + "\t" + replication + "\t" + length
                    + "\t" + path
            );
        }
    }

9.查看文件块信息

@Test
    public void getFileBlockLocations() throws Exception {

        FileStatus fileStatus = fileSystem.getFileStatus(new Path("/hdfsapi/test/jdk.tgz"));
        BlockLocation[] blocks = fileSystem.getFileBlockLocations(fileStatus,0,fileStatus.getLen());

        for(BlockLocation block : blocks) {

            for(String name: block.getNames()) {
                System.out.println(name +" : " + block.getOffset() + " : " + block.getLength() + " : " + block.getHosts());
            }
        }
    }

10删除文件

@Test
    public void delete() throws Exception {
        boolean result = fileSystem.delete(new Path("/hdfsapi/test/jdk.tgz"), true);
        System.out.println(result);
    }