java api 操作 hdfs

文章地址 ：http://www.haha174.top/article/details/255189

1.简介

之前说到了shell 脚本操作hdfs 现在那么本篇文章讲述java 怎么操作hdfs
使用 sh start-dfs.sh 启动 hdfs jps 查看hdfs 状态 确保 hdfs
SecondaryNameNode NameNode DataNode 都已经 启动（如果没有安装hdfs 可以参考这篇 http://blog.csdn.net/u012957549/article/details/78787411）

2.创建工程

使用idea 创建一个maven 工程
编写依赖

 <dependencies>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>3.0.0-alpha1</version>
        </dependency>

        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.12</version>
            <scope>test</scope>
        </dependency>
    </dependencies>

3. 编写

3.1 before 和after
junit 提供一个 before 和 after 的注解
before 在测试代码 执行之前生效
after 在测试代码执行结束后生效
3.1.1 before 把链接配置 放在 before 中

    public static final String HDFS_PATH = "hdfs://192.168.1.223:8020";
    FileSystem fileSystem = null;
    Configuration configuration = null;

  @Before
    public void setUp() throws Exception {
        System.out.println("HDFSApp.setUp");
        configuration = new Configuration();
        fileSystem = FileSystem.get(new URI(HDFS_PATH), configuration, "root");
    }

使用 这种方法 填入url 和user 获取远程配置
3.1.2 after

    @After
    public void tearDown() throws Exception {
        configuration = null;
        fileSystem = null;

        System.out.println("HDFSApp.tearDown");
    }

3.2 创建HDFS目录

    @Test
    public void mkdir() throws Exception {
        fileSystem.mkdirs(new Path("/hdfsapi/test"));
    }

3.3 创建文件

    @Test
    public void create() throws Exception {
        FSDataOutputStream output = fileSystem.create(new Path("/hdfsapi/test/a.txt"));
        output.write("hello hadoop".getBytes());
        output.flush();
        output.close();
    }

3.4 查看HDFS文件的内容

    @Test
    public void cat() throws Exception {
        FSDataInputStream in = fileSystem.open(new Path("/hdfsapi/test/a.txt"));
        IOUtils.copyBytes(in, System.out, 1024);
        in.close();
    }

3.5 重命名

    @Test
    public void rename() throws Exception {
        Path oldPath = new Path("/hdfsapi/test/a.txt");
        Path newPath = new Path("/hdfsapi/test/b.txt");
        fileSystem.rename(oldPath, newPath);
    }

3.6 上传文件到HDFS

    @Test
    public void copyFromLocalFile() throws Exception {
        Path localPath = new Path("C:\\Users\\haha174\\Desktop\\work\\mesos.yaml");
        Path hdfsPath = new Path("/hdfsapi/test");
        fileSystem.copyFromLocalFile(localPath, hdfsPath);
    }

3.7 上传文件到HDFS

 @Test
    public void copyFromLocalFileWithProgress() throws Exception {
        InputStream in = new BufferedInputStream(
                new FileInputStream(
                        new File("C:\\haha174\\nginx\\html\\hadoop-3.0.0.tar.gz")));

        FSDataOutputStream output = fileSystem.create(new Path("/hdfsapi/test/hadoop-3.0.0.tar.gz"),
                new Progressable() {
                    public void progress() {
                        System.out.print(".");  //带进度提醒信息
                    }
                });


        IOUtils.copyBytes(in, output, 4096);
    }

3.8 下载HDFS文件

    @Test
    public void copyToLocalFile() throws Exception {
        Path localPath = new Path("c:/tmp/h.txt");
        Path hdfsPath = new Path("/hdfsapi/test/hello.txt");
        fileSystem.copyToLocalFile(hdfsPath, localPath);
    }

3.9 查看某个目录下的所有文件

    @Test
    public void listFiles() throws Exception {
        FileStatus[] fileStatuses = fileSystem.listStatus(new Path("/"));

        for(FileStatus fileStatus : fileStatuses) {
            String isDir = fileStatus.isDirectory() ? "文件夹" : "文件";
            short replication = fileStatus.getReplication();
            long len = fileStatus.getLen();
            String path = fileStatus.getPath().toString();

            System.out.println(isDir + "\t" + replication + "\t" + len + "\t" + path);
        }

3.10 删除

    @Test
    public void delete() throws Exception{
        fileSystem.delete(new Path("/"), true);
    }