Hadoop学习——hdfs客户端操作及hdfs的API 操作

首先要在ubuntu上运行hadoop集群环境
1.在hfds传建目录

    /**
     * 在hfds传建目录
     * @throws URISyntaxException
     * @throws IOException
     * @throws InterruptedException
     */
    @Test
    public  void testMkdirs() throws URISyntaxException, IOException, InterruptedException {
        Configuration conf = new Configuration();
        // 1 获取hdfs客户端对象
        FileSystem fs = FileSystem.get(new URI("hdfs://localhost:9000"), conf, "hadoop");

        // 2 在hdfs上创建路径
        fs.mkdirs(new Path("/0529/dashen/banzhang"));

        // 3 关闭资源
        fs.close();

        System.out.println("over");
    }

从本地将文件上传到hdfs

 /**
     * 1 文件上传
     * @throws URISyntaxException
     * @throws IOException
     * @throws InterruptedException
     */
    @Test
    public void testCopyFromLocalFile() throws URISyntaxException, IOException, InterruptedException {
        // 1 获取hdfs客户端对象
        Configuration conf = new Configuration();
        conf.set("dfs.replication", "2");// 权限
        FileSystem fs = FileSystem.get(new URI("hdfs://localhost:9000"), conf, "hadoop");

        // 2 执行上传API
        fs.copyFromLocalFile(new Path("/home/hadoop/MyTmp/test.txt"),new Path("/CopyFromLocalFile2.txt"));

        // 3 关闭资源
        fs.close();

        System.out.println("over");
    }

2 从hdfs将文件下载到本地

/**
     * 2 文件下载
     * @throws URISyntaxException
     * @throws IOException
     * @throws InterruptedException
     */
    @Test
    public void testCopyToLocal() throws URISyntaxException, IOException, InterruptedException {
        // 1 获取hdfs客户端对象
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(new URI("hdfs://localhost:9000"), conf, "hadoop");

        // 2 执行下载操作
//        fs.copyToLocalFile(new Path(), new Path());
        fs.copyToLocalFile(false,new Path("/CopyFromLocalFile.txt"),new Path("/home/hadoop/MyTmp/CopyToLocal.txt"),true);

        //3.close()
        fs.close();

        System.out.println("voer");
    }

将hdfs 的文件进行删除

 /**
     * 3 文件删除
     * @throws IOException
     * @throws URISyntaxException
     * @throws InterruptedException
     */
    @Test
    public void testDelete() throws IOException, URISyntaxException, InterruptedException {
        // 1 获取hdfs客户端对象
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(new URI("hdfs://localhost:9000"), conf, "hadoop");

        // 2. delete file
        fs.delete(new Path("/0529"),true);

        //3.close()
        fs.close();

        System.out.println("over");
    }

对hdfs 内的文件进行更名

/**
     * 4 文件更名
     * @throws URISyntaxException
     * @throws IOException
     * @throws InterruptedException
     */
    @Test
    public void testRename() throws URISyntaxException, IOException, InterruptedException {
        //1. 获取hdfs客户端对象
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(new URI("hdfs://localhost:9000"), conf, "hadoop");

        //2 rename
        fs.rename(new Path("/sanguo"),new Path("/三国 "));

        //3. close()
        fs.close();

        System.out.println("over");
    }

文件详情查看

 /**
     * 5 文件详情查看
     * @throws URISyntaxException
     * @throws IOException
     * @throws InterruptedException
     */
    @Test
    public void testListFiles() throws URISyntaxException, IOException, InterruptedException {
        //1. 获取hdfs客户端对象
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(new URI("hdfs://localhost:9000"), conf, "hadoop");

        // 2 查看文件详情
        RemoteIterator<LocatedFileStatus> listFiles = fs.listFiles(new Path("/"), true);

        while(listFiles.hasNext()){
            LocatedFileStatus fileStatus = listFiles.next();

            // 查看文件名称、权限、长度、块信息
            System.out.println(fileStatus.getPath().getName());// 文件名称
            System.out.println(fileStatus.getPermission());// 文件权限
            System.out.println(fileStatus.getLen());// 文件长度

            BlockLocation[] blockLocations = fileStatus.getBlockLocations();

            for (BlockLocation blockLocation : blockLocations) {

                String[] hosts = blockLocation.getHosts();

                for (String host : hosts) {
                    System.out.println(host);
                }
            }

            System.out.println("------分割线--------");
        }

        //3 close()
        fs.close();
    }

判断是文件还是文件夹

/**
     * 6 判断是文件还是文件夹
     * @throws URISyntaxException
     * @throws IOException
     * @throws InterruptedException
     */
    @Test
    public void testListStatus() throws URISyntaxException, IOException, InterruptedException {
        //1. 获取hdfs客户端对象
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(new URI("hdfs://localhost:9000"), conf, "hadoop");

        // 2 判断操作
        FileStatus[] listStatus = fs.listStatus(new Path("/"));

        for (FileStatus fileStatus : listStatus) {

            if (fileStatus.isFile()) {
                // 文件
                System.out.println("f:"+fileStatus.getPath().getName());
            }else{
                // 文件夹
                System.out.println("d:"+fileStatus.getPath().getName());
            }
        }
        // 3 关闭资源
        fs.close();
    }