java 操作hdfs文件

1 依赖

<dependency>
     <groupId>org.apache.hadoop</groupId>
     <artifactId>hadoop-common</artifactId>
     <version>3.1.0</version>
</dependency>
<dependency>
     <groupId>org.apache.hadoop</groupId>
     <artifactId>hadoop-hdfs</artifactId>
     <version>3.1.0</version>
</dependency>
<dependency>
     <groupId>org.apache.hadoop</groupId>
     <artifactId>hadoop-mapreduce-client-core</artifactId>
     <version>3.1.0</version>
</dependency>

2 demo演示

//文件系统
    private FileSystem fileSystem = null;
    //配置文件
    private Configuration configuration = null;
    //虚拟机的HDFS的访问URI fs.defaultFS
    private static final String HDFS_URI = "hdfs://127.0.0.1/8020";

    //开始前操作--初始化资源
    @Before
    public void setUp() throws Exception {
        configuration = new Configuration();
        fileSystem = FileSystem.get(new URI(HDFS_URI), configuration, "hdfs");
    }

    //结束后操作--释放资源
    @After
    public void clearTail() throws Exception {
        configuration = null;
        fileSystem = null;
    }


    @Test
    public void fangfa_1() throws Exception {
        //查看txt文件的内容 控制台输出
        String str="/user/oozie/oozie_test/dir1/file1.txt";
        FSDataInputStream open = fileSystem.open(new Path(str));
        IOUtils.copyBytes(open, System.out,1024);


        //创建目录
        String dir="/user/oozie/oozie_test/dir1/a_1";
        boolean is_mkdirs = fileSystem.mkdirs(new Path(dir));
        System.out.println(is_mkdirs);


        //创建txt文件,并写入数据
        String file1_str="/user/oozie/oozie_test/dir1/a_1/a.txt";
        FSDataOutputStreamBuilder file = fileSystem.createFile(new Path(file1_str));
        file.build().write("鸿蒙 OS 的“分布式 OS 架构”和“分布式软总线技术”通过公共通信平台，分布式数据管理，分布式能力调度和虚拟外设四大能力，将相应分布式应用的底层技术实现难度对应用开发者屏蔽，使开发者能够聚焦自身业务逻辑，像开发同一终端一样开发跨终端分布式应用，也使最终消费者享受到强大的跨终端业务协同能力为各使用场景带来的无缝体验".getBytes());


        //重命名文件
        String oldPath="/user/oozie/oozie_test/dir1/a_1/a.txt";
        String newPath="/user/oozie/oozie_test/dir1/a_1/a_0.txt";
        fileSystem.rename(new Path(oldPath),new Path(newPath));


        //上传文件到hdfs
        String from_Str="C:\\Users\\need\\Desktop\\xls_txt.rar";
        String to_str="/user/oozie/oozie_test/dir1/a_1";
        fileSystem.copyFromLocalFile(new Path(from_Str),new Path(to_str));


        //上传文件到hdfs--加进度条
        String upfile_str="C:\\Users\\need\\Desktop\\新建文件夹 (4)\\common_time_size.log._0";
        BufferedInputStream bufferedInputStream = new BufferedInputStream(new FileInputStream(upfile_str));
        FSDataOutputStream fsDataOutputStream = fileSystem.create(new Path("/user/oozie/oozie_test/dir1/a_1/log_2"), new Progressable() {
            @Override
            public void progress() {
                System.out.print("*");
            }
        });
        IOUtils.copyBytes(bufferedInputStream,fsDataOutputStream,1024);


        //下载hdfs文件
        String hdfs_txt="/user/oozie/oozie_test/dir1/a_1/a_0.txt";
        String win_txt="C:\\Users\\need\\Desktop\\新建文件夹 (4)\\11\\salesPay\\a_0.txt";
        fileSystem.copyToLocalFile(new Path(hdfs_txt),new Path(win_txt));


        //删除hdfs文件
        String pathstr="/user/oozie/oozie_test/dir1/a_1/xls_txt.rar";
        //true/false是否是用递归进行删除
        fileSystem.delete(new Path(pathstr),true);


        //查看某一个目录下的所有文件
        String dir_str="/user/oozie/oozie_test/dir1/";
        RemoteIterator<LocatedFileStatus> locatedFileStatusRemoteIterator = fileSystem.listFiles(new Path(dir_str), true);
        while (locatedFileStatusRemoteIterator.hasNext())
        {
            LocatedFileStatus next = locatedFileStatusRemoteIterator.next();
            String isDir = next.isDirectory() ? "文件夹" : "文件";
            //副本数
            short replication = next.getReplication();
            //大小
            long len = next.getLen();
            //路径
            String lujing = next.getPath().toString();
            System.out.println(lujing+"---"+isDir+"---"+len+"----"+replication);
        }


        //查看某一个目录下的所有文件和文件夹
        String dir_str="/user/oozie/oozie_test/dir1/";
        FileStatus[] fileStatuses = fileSystem.listStatus(new Path(dir_str));
        for (FileStatus fs:fileStatuses) {
            String isDir = fs.isDirectory() ? "文件夹" : "文件";
            //副本数
            short replication = fs.getReplication();
            //大小
            long len = fs.getLen();
            //路径
            String lujing = fs.getPath().toString();
            System.out.println(lujing+"---"+isDir+"---"+len+"----"+replication);
        }

        
        //hdfs文件复制 复制log_1到log_3
        String hdfsfile_str_1="/user/oozie/oozie_test/dir1/a_1/log_1";
        FSDataInputStream inputStream = fileSystem.open(new Path(hdfsfile_str_1));
        String hdfsfile_str_2="/user/oozie/oozie_test/dir1/a_1/log_3";
        FSDataOutputStream outputStream = fileSystem.create(new Path(hdfsfile_str_2));
        IOUtils.copyBytes(inputStream,outputStream,1024,false);


        //获取配置的所有信息
        Iterator<Map.Entry<String, String>> it = fileSystem.getConf().iterator();
        while(it.hasNext()){
            Map.Entry<String,String> entry = it.next();
            System.out.println(entry.getKey()+"=" +entry.getValue());
        }
    }