HDFS的JAVA API操作

public class HDFSApp {

    public static final String HDFS_PATH = "hdfs://192.168.228.100:9000";
    Configuration configuration;
    FileSystem fileSystem;

    @Before
    public void setUp() throws Exception {
        //configuration可以保持默认；
        // 也可以在这里一行一行写配置；
        // 也可以将hdfs-site.xml文件复制到resources文件夹（新创建）下，并"Mark Directory as" -> "Resources Root"。
        configuration = new Configuration();
        configuration.set("dfs.replication", "1");
        fileSystem = FileSystem.get(new URI(HDFS_PATH), configuration, "hadoop");
    }

    @After
    public void tearDown() throws Exception {
        fileSystem.close();
    }

    @Test
    public void mkdir() throws Exception {
        boolean isSuccess = fileSystem.mkdirs(new Path("/ruozedata/hdfsapi"));
        //断言中写期望的值和实际的值
        Assert.assertEquals(true, isSuccess);
    }

    @Test
    public void copyFromLocalFile() throws Exception {
        Path srcPath = new Path("c:/kms10.log");
        Path dstPath = new Path("/ruozedata/hdfsapi");
        fileSystem.copyFromLocalFile(srcPath, dstPath);
    }

    @Test
    public void testReplication() {
        System.out.println(configuration.get("dfs.replication"));
    }

    @Test
    public void copyToLocalFile() throws Exception {
        Path srcPath = new Path("/ruozedata/hdfsapi/kms10.log");
        Path dstPath = new Path("D:/spark_code/test_data/kms10.log");
        fileSystem.copyToLocalFile(srcPath, dstPath);
    }

    @Test
    public void rename() throws Exception {
        Path srcPath = new Path("/ruozedata/hdfsapi/kms10.log");
        Path dstPath = new Path("/ruozedata/hdfsapi/kms10-2.log");
        fileSystem.rename(srcPath, dstPath);
    }

    @Test
    public void listFile() throws Exception {
        RemoteIterator<LocatedFileStatus> files = fileSystem.listFiles(new Path("/ruozedata"), true);
        while (files.hasNext()) {
            LocatedFileStatus fileStatus = files.next();
            String isDir = fileStatus.isDirectory() ? "文件夹" : "文件";
            String Permission = fileStatus.getPermission().toString();
            short replication = fileStatus.getReplication();
            long len = fileStatus.getLen();
            String path = fileStatus.getPath().toString();
            System.out.println(isDir + "\t" + Permission + "\t" + replication + "\t" + len + "\t" + path);

            BlockLocation[] blockLocations = fileStatus.getBlockLocations();
            for (BlockLocation location : blockLocations) {
                String[] hosts = location.getHosts();
                for (String host : hosts) {
                    System.out.println(host);
                }
            }
            //可以看出无论文件的实际大小，块大小都是128M
            long blockSize = fileStatus.getBlockSize();
            System.out.println(blockSize);
        }
    }

    @Test
    public void copyFromLocalIO() throws Exception {
        //IO 字节流 字符流
        BufferedInputStream in = new BufferedInputStream(new FileInputStream(new File("D:/spark_code/test_data/kms10.log")));
        FSDataOutputStream out = fileSystem.create(new Path("/ruozedata/hdfsapi/kms10-io.log"));

        IOUtils.copyBytes(in, out, 4096, true);
//        IOUtils.closeStream(out);
//        IOUtils.closeStream(in);
    }

    @Test
    public void download01() throws Exception {

        FSDataInputStream in = fileSystem.open(new Path("/ruozedata/hdfsapi/spark-2.4.2-bin-2.6.0-cdh5.16.1.tgz"));
        FileOutputStream out = new FileOutputStream(new File("D:/spark_code/test_data/spark.tgz.part2"));

        //前几块可以用seek指定偏移量，然后buffer传输，但是最后一块不行，不然大小也是128M，最后一块只能通过IOUtils.copyBytes + seek方式
        in.seek(1024 * 1024 * 128 * 2);

//        byte[] buffer = new byte[1024];
//        for (int i = 0; i < 1024 * 128; i++) {
//            in.read(buffer);
//            out.write(buffer);
//        }

        IOUtils.copyBytes(in, out, configuration);
        IOUtils.closeStream(out);
        IOUtils.closeStream(in);
    }
}