JAVA API操作HDFS

JAVA API操作HDFS

注意事项

此处配置需相同

代码如下：

导入依赖
注：这里hadoop的版本最好与安装的hadoop版本一致

pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.gohb</groupId>
    <artifactId>hdfstest</artifactId>
    <version>1.0-SNAPSHOT</version>

    <dependencies>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>2.7.5</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-hdfs</artifactId>
            <version>2.7.5</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>2.7.5</version>
            <exclusions>
                <exclusion>
                    <groupId>log4j</groupId>
                    <artifactId>log4j</artifactId>
                </exclusion>
            </exclusions>
        </dependency>
        <dependency>
            <groupId>log4j</groupId>
            <artifactId>log4j</artifactId>
            <version>1.2.17</version>
        </dependency>

        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.12</version>
        </dependency>
    </dependencies>

</project>

log4j.properties

log4j.rootLogger=debug, stdout, R

log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout

log4j.appender.stdout.layout.ConversionPattern=%5p - %m%n

log4j.appender.R=org.apache.log4j.RollingFileAppender
log4j.appender.R.File=firestorm.log

log4j.appender.R.MaxFileSize=100KB
log4j.appender.R.MaxBackupIndex=1

log4j.appender.R.layout=org.apache.log4j.PatternLayout
log4j.appender.R.layout.ConversionPattern=%p %t %c - %m%n

log4j.logger.com.codefutures=DEBUG

测试

package com.gohb;


import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.io.IOUtils;
import org.junit.Test;


import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.OutputStream;


public class test {

    /**
     * 创建文件夹
     * @Author gohb
     */
    @Test
    public void mkDir()throws Exception{
        Configuration conf = new Configuration();
        conf.set("fs.default.name","hdfs://localhost:9000");
        FileSystem fs = FileSystem.get(conf);
        boolean res = fs.mkdirs(new Path("/localData"));
        if (res){
            System.out.println("文件夹创建成功");
        }else {
            System.out.println("文件夹创建失败");
        }
        fs.close();
    }

    /**
     * 上传文件
     * @Author gohb
     */
    @Test
    public void uploadFile()throws Exception{
        Configuration conf = new Configuration();
        conf.set("fs.default.name","hdfs://localhost:9000");
        FileSystem fs = FileSystem.get(conf);

        String filename = "core-site.xml";
        InputStream in = new FileInputStream("f:\\hadoop-2.7.5\\etc\\hadoop\\"+filename);
        OutputStream out = fs.create(new Path("/localData/"+filename));
        IOUtils.copyBytes(in,out,4096,true);
        System.out.println("文件上传成功");

        fs.close();

    }

    /**
     * 下载文件
     * @Author gohb
     */
    @Test
    public void downloadFile()throws Exception{
        Configuration conf = new Configuration();
        conf.set("fs.default.name","hdfs://localhost:9000");
        FileSystem fs = FileSystem.get(conf);

        String filename = "core-site.xml";
        InputStream in = fs.open(new Path("/localData/"+filename));
        OutputStream out = new FileOutputStream("f:\\hadoop-2.7.5\\"+filename);
        IOUtils.copyBytes(in,out,4096,true);
        System.out.println("文件下载成功");

        fs.close();

    }

    /**
     * 删除文件
     * @Author gohb
     */
    @Test
    public void deleteFile()throws Exception{
        Configuration conf = new Configuration();
        conf.set("fs.default.name","hdfs://localhost:9000");
        FileSystem fs = FileSystem.get(conf);

        String filename = "core-site.xml";
        boolean res = fs.delete(new Path("/localData/" + filename), true);
        if(res){
            System.out.println("文件删除成功");
        }else {
            System.out.println("文件删除失败");
        }

        fs.close();
    }

    /**
     * 遍历文件
     * @Author gohb
     */
    @Test
    public void listFiles()throws Exception{
        Configuration conf = new Configuration();
        conf.set("fs.default.name","hdfs://localhost:9000");
        FileSystem fs = FileSystem.get(conf);

        RemoteIterator<LocatedFileStatus> locatedFileStatusRemoteIterator = fs.listFiles(new Path("/"), true);
        while (locatedFileStatusRemoteIterator.hasNext()){
            System.out.println(locatedFileStatusRemoteIterator.next());
        }

        fs.close();
    }

}