HDFS的API操作

导入依赖

    <dependencies>
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-common</artifactId>
        <version>2.7.5</version>
    </dependency>
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-client</artifactId>
        <version>2.7.5</version>
    </dependency>
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-hdfs</artifactId>
        <version>2.7.5</version>
    </dependency>
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-mapreduce-client-core</artifactId>
        <version>2.7.5</version>
    </dependency>
    <dependency>
        <groupId>junit</groupId>
        <artifactId>junit</artifactId>
        <version>RELEASE</version>
    </dependency>
    <dependency>
        <groupId>junit</groupId>
        <artifactId>junit</artifactId>
        <version>4.0</version>
    </dependency>
    <dependency>
        <groupId>junit</groupId>
        <artifactId>junit</artifactId>
        <version>4.0</version>
    </dependency>
</dependencies>
<build>
    <plugins>
        <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-compiler-plugin</artifactId>
            <version>3.1</version>
            <configuration>
                <source>1.8</source>
                <target>1.8</target>
                <encoding>UTF-8</encoding>
                <!--    <verbal>true</verbal>-->
            </configuration>
        </plugin>
        <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-shade-plugin</artifactId>
            <version>3.2.0</version>
            <executions>
                <execution>
                    <phase>package</phase>
                    <goals>
                        <goal>shade</goal>
                    </goals>
                    <configuration>
                        <minimizeJar>true</minimizeJar>
                    </configuration>
                </execution>
            </executions>
        </plugin>

    </plugins>
</build>




 package com.domian;
/*利用javaapi测试hdfs的命令*/
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.junit.Test;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;


public class Hdfs {


//使用url方式访问数据]
@Test
public void testUrl() throws IOException {
    //第一步：注册hdfs 的url
    URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory());
    //获取文件输入流
    InputStream inputStream = new URL("hdfs://node01:8020/1.txt").openStream();
    //获取文件输出流
    FileOutputStream outputStream = new FileOutputStream(new File("D:\\input\\a.txt"));
    //利用IOUTILS工具类进行拷贝
    IOUtils.copy(inputStream,outputStream);

    //关闭流
    IOUtils.closeQuietly(inputStream);
    IOUtils.closeQuietly(outputStream);
}

//获取 FileSystem 的第一种方式
@Test
public void testFileSystem() throws IOException {

    Configuration configuration = new Configuration();
    //指定我们使用的文件系统类型
    configuration.set("fs.defaultFS","hdfs://node01:8020/");
    //获取指定的文件系统
    FileSystem fileSystem = FileSystem.get(configuration);
    System.out.println(fileSystem.toString());
}

//获取FileSystem 的第二种方式
@Test
public void testFileSystemto() throws Exception {
    FileSystem fileSystem = FileSystem.get(new URI("hdfs://node01:8020"),new Configuration());
    System.out.println(fileSystem);
}

//获取FilSystem 的第三种方式
@Test
public void testFileSystemtwo() throws IOException {
    Configuration configuration = new Configuration();
    configuration.set("fs.defaultFS","hdfs://node01:8020");
    FileSystem fileSystem = FileSystem.newInstance(configuration);
    System.out.println(fileSystem.toString());
}

//获取FileSystem方式
@Test
public void getFileSystem() throws URISyntaxException, IOException {
    FileSystem fileSystem = FileSystem.newInstance(new URI("hdfs://node01:8020"), new Configuration());
    System.out.println(fileSystem.toString());
}

//遍历HDFS中的所有文件
@Test
public void listMyFiles() throws URISyntaxException, IOException {
    //获取FileSystem类
    FileSystem fileSystem = FileSystem.get(new URI("hdfs://node01:8020"), new Configuration());
    RemoteIterator<LocatedFileStatus> locatedFileStatusRemoteIterator = fileSystem.listFiles(new Path("/"), true);

    while (locatedFileStatusRemoteIterator.hasNext()){
        LocatedFileStatus next = locatedFileStatusRemoteIterator.next();
        System.out.println(next.getPath().toString());
    }
    fileSystem.close();
}

// HDFS 上创建文件夹
@Test
public void testmkdirs() throws URISyntaxException, IOException, InterruptedException {
    FileSystem fileSystem = FileSystem.get(new URI("hdfs://node01:8020"), new Configuration(),"root");
    boolean b = fileSystem.mkdirs(new Path("/mulu"));
    System.out.println(b);
}


//从hdfs上下载文件到本地
@Test
public void testFileToLocal() throws URISyntaxException, IOException, InterruptedException {
    //获取FileSystem方式
    FileSystem fileSystem = FileSystem.get(new URI("hdfs://node01:8020"), new Configuration(),"root");
    //获取输入流
    FSDataInputStream fsDataInputStream = fileSystem.open(new Path("hdfs://node01:8020//1.txt"));
    //输出流
    FileOutputStream fileOutputStream = new FileOutputStream(new File("D://input//1.txt"));

    //利用IOUtils工具类
    int i = IOUtils.copy(fsDataInputStream, fileOutputStream);
    System.out.println(i);
    fileOutputStream.close();
    fsDataInputStream.close();
    fileSystem.close();
}

从hdfs上下载文件到本地的第二种方式
@Test
public void testFileToLocal2() throws Exception {
    FileSystem fileSystem = FileSystem.get(new URI("hdfs://node01:8020//"), new Configuration(), "root");

    fileSystem.copyToLocalFile(new Path("hdfs://node01:8020//1.txt"),new Path("D:\\input\\1.txt"));
    fileSystem.close();
}

//从本地上传到hdfs上
@Test
public void testputData() throws Exception {
    FileSystem fileSystem = FileSystem.get(new URI("hdfs://node01:8020//"), new Configuration(), "root");
    fileSystem.copyFromLocalFile(new Path("D:\\input\\san.txt"),new Path("hdfs://node01:8020//"));
    fileSystem.close();
}

//从本地多个文件上传到hdfs一个文件上
@Test
public void testmergeFile() throws Exception {
    FileSystem fileSystem = FileSystem.get(new URI("hdfs://node01:8020//"), new Configuration(), "root");

    //获取hdfs上的输出流
    FSDataOutputStream fsDataOutputStream = fileSystem.create(new Path("hdfs://node01:8020//big.txt"));
    //获取本地的文件系统
    LocalFileSystem local = FileSystem.getLocal(new Configuration());
    //通过本地文件系统获取文件列表，为一个集合
    FileStatus[] fileStatuses = local.listStatus(new Path("D:\\input"));

    for (FileStatus fileStatus : fileStatuses) {
        //然后输入流
        FSDataInputStream fsDataInputStream = local.open(fileStatus.getPath());
        //把本地的拷贝到hdfs上
        int i = IOUtils.copy(fsDataInputStream, fsDataOutputStream);
        System.out.println(i);
        fsDataInputStream.close();
    }
    fsDataOutputStream.close();
    fileSystem.close();
}

//把多个hdfs上的文件下载到本地一个文件上
@Test
public void testgetmerge() throws Exception {
    //获得HDFS的文件系统
    FileSystem fileSystem = FileSystem.get(new URI("hdfs://node01:8020//"), new Configuration(), "root");
    //获取本地的文件系统
    LocalFileSystem local = FileSystem.getLocal(new Configuration());
    //获取本地的输出流
    FSDataOutputStream dataOutputStream = local.create(new Path("D:\\input\\55.txt"));

    //获得hdfs的输入文件，为一个集合
    FileStatus[] fileStatuses1 = fileSystem.listStatus(new Path("hdfs://node01:8020//mulu"));

    for (FileStatus fileStatus : fileStatuses1) {
        //然后输入流
        FSDataInputStream fsDataInputStream = fileSystem.open(fileStatus.getPath());
        //把hdfs的拷贝到本地上
        int i = IOUtils.copy(fsDataInputStream, dataOutputStream);
        System.out.println(i);
        fsDataInputStream.close();
    }
    dataOutputStream.close();
    fileSystem.close();
}
}