HDFS的API使用_学习

前提，部署好windows本地的hadoop环境及环境变量

• 本文创建的是Maven项目，导入依赖包操作
• maven配置

<dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>2.7.7</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>2.7.7</version>
        </dependency>

        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-hdfs</artifactId>
            <version>2.7.7</version>
            <scope>test</scope>
        </dependency>

        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.12</version>
        </dependency>

• 测试代码

package hdfs;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;
import org.junit.Before;
import org.junit.Test;

import java.io.FileInputStream;
import java.io.IOException;

/**
 * @ClassName HdfsUtils
 * @Description TODO
 * @Author fengle
 * @Date 2021/6/15 21:04
 * @Version 1.0
 */
public class HdfsUtils {
    FileSystem fs = null;
	
    @Before
    public void init() throws IOException {
        Configuration conf = new Configuration();
        // 可以配置，也可将xml文件导入项目src下,配置会覆盖xml(core-site.xml,hdfs-site.xml)
        conf.set("fs.defaultFS", "hdfs://hadoop1001:9000");
        // 设置hadoop访问用户，本地windows为admin
        System.setProperty("HADOOP_USER_NAME", "root");
        // 获取文件系统实例
        fs = FileSystem.get(conf);
    }

    @Test
    public void upload() throws IOException {
        // 上传文件，存在的文件路径
        // 也可以用 fs.copyFromLocalFile();
        Path path = new Path("hdfs://hadoop1001:9000/aaa/bbb/ccc/hello.txt");
        FSDataOutputStream os = fs.create(path);
        FileInputStream in = new FileInputStream("g:/hello.txt");
        IOUtils.copyBytes(in, os, 100);
    }

    @Test
    public void downLoad() throws IOException {
        // 下载文件，源目录，本地目录
        fs.copyToLocalFile(new Path("hdfs://hadoop1001:9000/hello"), new Path("g:/hello2.txt"));
    }

    @Test
    public void mkdir() throws IOException {
        // 创建文件夹
        fs.mkdirs(new Path("/aaa/bbb/ccc"));
    }

    @Test
    public void rm() throws IOException {
        // 删除文件夹，第二参数是否递归删除
        fs.delete(new Path("/aaa"), true);
    }


    @Test
    public void listFiles() throws IOException {
        // 列出所有文件
        RemoteIterator<LocatedFileStatus> remoteIterator = fs.listFiles(new Path("/"), true);
        while (remoteIterator.hasNext()) {
            LocatedFileStatus file = remoteIterator.next();
            Path path = file.getPath();
            String name = path.getName();
            System.out.println(name);
        }

        System.out.println("=============================================");

        FileStatus[] statuses = fs.listStatus(new Path("/"));
        for (FileStatus file : statuses) {
            String name = file.getPath().getName();
            System.out.println(name + (file.isDirectory() ? " is Dir" : " is File"));
        }
    }
}