HDFS API常用的方法及使用

POM依赖

<repositories>
    <repository>
        <id>cloudera</id>
        <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
    </repository>
</repositories>
<dependencies>
    <dependency>
        <groupId>org.apache.Hadoop</groupId>
        <artifactId>Hadoop-client</artifactId>
        <version>2.6.0-mr1-cdh5.14.0</version>
    </dependency>
    <dependency>
        <groupId>org.apache.Hadoop</groupId>
        <artifactId>Hadoop-common</artifactId>
        <version>2.6.0-cdh5.14.0</version>
    </dependency>
    <dependency>
        <groupId>org.apache.Hadoop</groupId>
        <artifactId>Hadoop-hdfs</artifactId>
        <version>2.6.0-cdh5.14.0</version>
    </dependency>

    <dependency>
        <groupId>org.apache.Hadoop</groupId>
        <artifactId>Hadoop-mapreduce-client-core</artifactId>
        <version>2.6.0-cdh5.14.0</version>
    </dependency>
    <!-- https://mvnrepository.com/artifact/junit/junit -->
    <dependency>
        <groupId>junit</groupId>
        <artifactId>junit</artifactId>
        <version>4.11</version>
        <scope>test</scope>
    </dependency>
    <dependency>
        <groupId>org.testng</groupId>
        <artifactId>testng</artifactId>
        <version>RELEASE</version>
    </dependency>
</dependencies>
<build>
    <plugins>
        <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-compiler-plugin</artifactId>
            <version>3.0</version>
            <configuration>
                <source>1.8</source>
                <target>1.8</target>
                <encoding>UTF-8</encoding>
                <!--    <verbal>true</verbal>-->
            </configuration>
        </plugin>

        <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-shade-plugin</artifactId>
            <version>2.4.3</version>
            <executions>
                <execution>
                    <phase>package</phase>
                    <goals>
                        <goal>shade</goal>
                    </goals>
                    <configuration>
                        <minimizeJar>true</minimizeJar>
                    </configuration>
                </execution>
            </executions>
        </plugin>
      <!--  <plugin>
            <artifactId>maven-assembly-plugin </artifactId>
            <configuration>
                <descriptorRefs>
                    <descriptorRef>jar-with-dependencies</descriptorRef>
                </descriptorRefs>
                <archive>
                    <manifest>
                        <mainClass>cn.itcast.Hadoop.db.DBToHdfs2</mainClass>
                    </manifest>
                </archive>
            </configuration>
            <executions>
                <execution>
                    <id>make-assembly</id>
                    <phase>package</phase>
                    <goals>
                        <goal>single</goal>
                    </goals>
                </execution>
            </executions>
        </plugin>-->
    </plugins>
</build>

HDFS API

上传本地数据到HDFS系统

copyFromLocalFile()

	/**
     * 上传本地数据到HDFS系统
     */
    @Test
    public void copyFromLocalFile() throws Exception {
        Configuration configuration = new Configuration();
        FileSystem fileSystem = FileSystem.get(new URI("hdfs://192.168.100.21/"), configuration);
        fileSystem.copyFromLocalFile(new Path("D://123.txt"), new Path("/"));

        fileSystem.close();
    }

删除指定路径下的文件

delete()

    /**
     * 删除指定路径下的文件
     */
    public void deleteFile() throws Exception {
        Configuration configuration = new Configuration();
        FileSystem fileSystem = FileSystem.get(new URI("hdfs://192.168.100.21/"), configuration);
        Path rmPath = new Path("/abc/a.txt");
        /**
         * recursive 是否进行递归删除该文件夹下所有的文件？
         * true：是
         * false：否
         */
        boolean delete = fileSystem.delete(rmPath, true);

        System.out.println(delete ? "文件删除成功" : "文件删除失败");
        fileSystem.close();
    }

创建文件夹及文件，并向文件中写入数据

mkdir()
create()

    /**
     * 创建文件夹/abc和a.txt文件，并向a.txt文件中写入数据
     */
    public void createData() throws Exception {
        Configuration configuration = new Configuration();
        FileSystem fileSystem = FileSystem.get(new URI("hdfs://192.168.100.21"), configuration);
        FSDataOutputStream fsDataOutputStream = null;
        // 检查要创建的文件夹是否已在文件系统中存在
        Path dir = new Path("/abc");
        if (!fileSystem.exists(dir)) {
            fileSystem.mkdirs(dir);
        }

        // 检查要创建的文件是否已在文件系统中存在
        Path file = new Path(dir, "/a.txt");
        if (!fileSystem.exists(file)) {
            fsDataOutputStream = fileSystem.create(file, true, 4096, new Progressable() {
                @Override
                public void progress() {
                    System.out.println("显示创建进度（可有可无）");
                }
            });

        }
        
		byte[] bytes = "向文件中写入的数据内容".getBytes();

        fsDataOutputStream.write(bytes, 0, bytes.length);
        fsDataOutputStream.flush();
        fsDataOutputStream.close();
        fileSystem.close();
    }

获取指定文件的日期

getFileStatus()
getModificationTime()

    /**
     * 获取指定文件的日期
     */
    public void getTime() throws Exception {
        Configuration configuration = new Configuration();
        FileSystem fileSystem = FileSystem.get(new URI("hdfs://192.168.100.21/"), configuration);
        FileStatus fileStatus = fileSystem.getFileStatus(new Path("/a.txt"));
        long modificationTime = fileStatus.getModificationTime();

        System.out.println(modificationTime);
        fileSystem.close();
    }

重命名目录

rename()

	/**
     * 重命名目录名
     */
    public void rename() throws Exception {
        Configuration configuration = new Configuration();
        FileSystem fileSystem = FileSystem.get(new URI("hdfs://192.168.100.21/"), configuration);
        Path oldPath = new Path("/abc");
        Path newPath = new Path("/123");

        boolean rename = fileSystem.rename(oldPath, newPath);
        System.out.println(rename ? "重命名成功" : "重命名失败");
        fileSystem.close();
    }

获取指定路径所有的文件

listStatus()

	/**
     * 获取指定路径所有的文件（HDFS文件系统中根路径的所有文件）
     */
    public void listStatus() throws Exception {
        Configuration configuration = new Configuration();
        FileSystem fileSystem = FileSystem.get(new URI("hdfs://192.168.100.21/"), configuration);
        FileStatus[] fileStatuses = fileSystem.listStatus(new Path("/"));
        
        for (FileStatus fileStatus : fileStatuses) {
            System.out.println(fileStatus.getPath().toString());
        }
        fileSystem.close();
    }