4. HDFS Java API使用

1. HDFS API

Java抽象类org.apache.hadoop.fs.FileSystem定义了hadoop的一个文件系统接口。该类是一个抽象类，通过以下两种静态工厂方法可以过去FileSystem实例：

public static FileSystem.get(Configuration conf) throws IOException 
public static FileSystem.get(URI uri, Configuration conf) throws IOException 

常见方法介绍：
public boolean mkdirs(Path f) throws IOException
一次性新建所有目录（包括父目录）， f是完整的目录路径。

public FSOutputStream create(Path f) throws IOException
创建指定path对象的一个文件，返回一个用于写入数据的输出流
create()有多个重载版本，允许我们指定是否强制覆盖已有的文件、文件备份数量、写入文件缓冲区大小、文件块大小以及文件权限。

public boolean copyFromLocal(Path src, Path dst) throws IOException
将本地文件拷贝到HDFS文件系统

public boolean exists(Path f) throws IOException
检查文件或目录是否存在

public boolean delete(Path f, Boolean recursive)
永久性删除指定的文件或目录，如果f是一个空目录或者文件，那么recursive的值就会被忽略。只有recursive＝true时，一个非空目录及其内容才会被删除。

FileStatus类封装了文件系统中文件和目录的元数据，包括文件长度、块大小、备份、修改时间、所有者以及权限信息。

2. 例程

package com.tongfang.learn.hdfs;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.Progressable;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.*;
import java.net.URI;

public class HdfsTest {

    private static final String HDFS_URL = "hdfs://192.168.1.160:9000";

    FileSystem fileSystem = null;
    Configuration configuration = null;

    /**
     * 创建hdfs目录
     * @throws IOException
     */
    @Test
    public void testMkdir() throws IOException {
        fileSystem.mkdirs(new Path("/hdfs/test"));
    }
    
    /**
     * 创建文件
     * @throws IOException
     */
    @Test
    public void testCreateFile() throws IOException {
        FSDataOutputStream out = fileSystem.create(new Path("/hdfs/test/a.txt"));
        out.write("hello, I am a file !!!".getBytes());
        out.flush();
        out.close();
    }

    /**
     * 追加文件
     * @throws IOException
     */
    @Test
    public void testAppendFile() throws IOException {
        FSDataOutputStream out = fileSystem.append(new Path("/hdfs/test/a.txt"));
        out.write("append to file".getBytes());
        out.flush();
        out.close();
    }

    /**
     * 查看文件内容
     * @throws IOException
     */
    @Test
    public void testCatFile() throws IOException {
        FSDataInputStream in = fileSystem.open(new Path("/hdfs/test/a.txt"));
        IOUtils.copyBytes(in, System.out, 1024);
        in.close();
    }


    /**
     * 重命名文件
     * @throws IOException
     */
    @Test
    public void testRename() throws IOException {
        Path oldPath = new Path("/hdfs/test/a.txt");
        Path newPath = new Path("/hdfs/test/b.txt");
        fileSystem.rename(oldPath, newPath);
    }


    /**
     * 删除文件
     */
    @Test
    public void testDelete() throws IOException {
        fileSystem.delete(new Path("/hdfs/test"), true);
    }


    /**
     * 上传本地文件到HDFS
     * @throws IOException
     */
    @Test
    public void testCopyFromLocal() throws IOException {
        Path localPath = new Path("D:\\input.txt");
        Path hdfsPath = new Path("/hdfs/input.txt");

        fileSystem.copyFromLocalFile(localPath, hdfsPath);
    }

    @Test
    public void testCopyFromLocalWithStream() throws IOException {
        InputStream in = new BufferedInputStream(new FileInputStream("D:\\input.zip"));
        FSDataOutputStream output = fileSystem.create(new Path("/hdfs/input.zip"), new Progressable() {
            @Override
            public void progress() {
                System.out.print(".");
            }
        });

        IOUtils.copyBytes(in, output, 4096);
    }


    /**
     * 下载HDFS文件到本地
     * @throws IOException
     */
    @Test
    public void testCopyToLocal() throws IOException {
        Path localPath = new Path("D:\\local.txt");
        Path hdfsPath = new Path("/hdfs/input.txt");
        fileSystem.copyToLocalFile(false, hdfsPath, localPath, true);
    }

    /**
     * 下载Hdfs文件到本地
     */
    @Test
    public void testCopyToLocalWithStream() throws IOException {

        InputStream input = fileSystem.open(new Path("/hdfs/input.zip"));
        OutputStream output = new FileOutputStream("D:\\input.zip");
        IOUtils.copyBytes(input, output, 4096, true);
    }


    /**
     * 查看某个目录下的所有文件
     * @throws IOException
     */
    @Test
    public void testListFiles() throws IOException {
        FileStatus[] statuses = fileSystem.listStatus(new Path("/hdfs"));

        for (FileStatus status : statuses) {
            String isDir = status.isDirectory() ? "目录" : "文件";
            short replication = status.getReplication();
            long len = status.getLen();
            String path = status.getPath().toString();
            long modtime = status.getModificationTime();

            System.out.println(isDir + " " + replication + " " + len + " " +
                    path + " " + modtime);
        }

    }

    /**
     * 获取某个文件各个块在集群主机的分布信息
     * @throws IOException
     */
    @Test
    public void testFileBlockLocation() throws IOException {
        FileStatus status = fileSystem.getFileStatus(new Path("/hdfs/input.zip"));

        BlockLocation[] locations = fileSystem.getFileBlockLocations(status, 0, status.getLen());
        for (BlockLocation location: locations) {
            String[] hosts = location.getHosts();
            String allHosts = "";
            for (String host : hosts) {
                allHosts += host;
                allHosts += " ";
            }
            System.out.println("block: " + location + " hosts: " + allHosts);
        }
    }


    @Before
    public void setUp() throws Exception {
        configuration = new Configuration();
        //必须添加这两个配置，否则可能导致append文件失败
        configuration.set("dfs.client.block.write.replace-datanode-on-failure.policy","NEVER");
        configuration.set("dfs.client.block.write.replace-datanode-on-failure.enable", "true");
        fileSystem = FileSystem.get(new URI(HDFS_URL), configuration, "hadoop");
        System.out.println("testcase setup");
    }


    @After
    public void tearDown() {
        configuration = null;
        fileSystem = null;
        System.out.println("testcase teardown");
    }

}

maven 依赖

<dependency>
	<groupId>org.apache.hadoop</groupId>
	<artifactId>hadoop-client</artifactId>
	<version>3.2.4</version>
</dependency>

<dependency>
	<groupId>junit</groupId>
	<artifactId>junit</artifactId>
	<version>4.11</version>
	<scope>test</scope>
</dependency>