javaAPI操作hadoop hdfs

写在之前

在开始操作之前请确保已经正确安装启动hadoop并且能够连接到

依赖

<dependencies>
    <dependency>
        <groupId>junit</groupId>
        <artifactId>junit</artifactId>
        <version>4.13</version>
    </dependency>
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-client</artifactId>
        <version>3.0.3</version>
    </dependency>
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-common</artifactId>
        <version>3.0.3</version>
    </dependency>
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-hdfs</artifactId>
        <version>3.0.3</version>
    </dependency>
</dependencies>

读取文件

public void test1() throws Exception{
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(new URI("hdfs://192.168.19.4:9000"), conf);
        InputStream in = fs.open(new Path("/park/test.txt"));
        OutputStream out = new FileOutputStream("test.txt");
        IOUtils.copyBytes(in,out,conf);

}

写文件到hdfs

public void test2() throws Exception{
        Configuration conf = new Configuration();
        //设置副本数
        conf.set("dfs.replication","1");
        FileSystem fs = FileSystem.get(new URI("hdfs://192.168.19.4:9000"),conf,"root");
        ByteArrayInputStream in = new ByteArrayInputStream("helloworld".getBytes());
        OutputStream out = fs.create(new Path("/park/hello.txt"));
        IOUtils.copyBytes(in,out,conf);
}

删除文件

public void test3() throws Exception{
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(new URI("hdfs://192.168.19.4:9000"),conf,"root");
        fs.delete(new Path("park/hello.txt"),true);
        fs.close();

}

创建文件

public void test4() throws Exception{
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(new URI("hdfs://192.168.19.4:9000"),conf,"root");
        fs.mkdirs(new Path("/hello"));
        fs.close();
}

查询指定目录

public void test5() throws Exception{
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(new URI("hdfs://192.168.19.4:9000"),conf,"root");
        FileStatus[] ls = fs.listStatus(new Path("/"));//查询hdfs根目录
        for (FileStatus l : ls) {
                System.out.println(l.getPath());
        }
}

递归查看指定目录下的所有文件

public void test6() throws Exception{
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(new URI("hdfs://192.168.19.4:9000"),conf,"root");
        RemoteIterator<LocatedFileStatus> rt = fs.listFiles(new Path("/"),true);
        while (rt.hasNext()){
                System.out.println(rt.next());
        }
}

重命名

public void test7() throws Exception{
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(new URI("hdfs://192.168.19.4:9000"),conf,"root");
        fs.rename(new Path("/park"),new Path("/park1"));
}

获取文件块信息

public void test8() throws Exception{
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(new URI("hdfs://192.168.19.4:9000"),conf,"root");
        BlockLocation[] data = fs.getFileBlockLocations(new Path("/park1/hello.txt"),0,Integer.MAX_VALUE);
        for (BlockLocation datum : data) {
                System.out.println(datum);
        }

}