Java操作HDFS对象

1.使用Eclipse工具创建一个maven工程，配置pom.xml

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>

  <groupId>com.sl</groupId>
  <artifactId>bigdatatest</artifactId>
  <version>0.0.1-SNAPSHOT</version>
  <packaging>war</packaging>

  <name>bigdatatest</name>
  <url>http://maven.apache.org</url>

  <properties>
    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
  </properties>

  <dependencies>
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-common</artifactId>
        <version>2.6.4</version>
    </dependency>
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-client</artifactId>
        <version>2.6.4</version>
    </dependency>
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-hdfs</artifactId>
        <version>2.6.4</version>
    </dependency>
    <dependency>
        <groupId>junit</groupId>
        <artifactId>junit</artifactId>
        <version>3.8.1</version>
        <scope>test</scope>
    </dependency>
  </dependencies>
</project>

2. Java操作HDFS对象

2.1获取根目录下所有文件信息

import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

/**  
 * @description  Java操作HDFS对象
 *   得到DFSClient对象并得到HDFS根目录下所有对象
 *   FileSystem对象就是DFSClient对象，也就是描述HDFS文件系统的一个对象。
 */
public class FileSystemOperate {

    FileSystem fileSystem;
    Configuration conf;

    /**
     * URI对象通过使用HDFS协议指定HDFS文件系统配置的节点地址
     * URI中的ip地址可以换成主机名但是需要配置好ip与主机的映射关系
     * @throws URISyntaxException
     * @throws IOException
     */
    @Before
    public void before() throws URISyntaxException, IOException {
        //1.得到DFSClient对象
        conf = new Configuration();
        URI uri = new URI("hdfs://192.168.1.200:9000");
        fileSystem = FileSystem.get(uri, conf);
        System.out.println(fileSystem);
    }

    @After
    public void after() throws IOException {
        fileSystem.close();
    }

    @Test
    public void getFileStatus() throws FileNotFoundException, IllegalArgumentException, IOException {
        //2.获取路径根目录下的所有对象信息
        FileStatus[] listStatus = fileSystem.listStatus(new Path("/"));
        for (FileStatus fileStatus : listStatus) {
            System.out.println(fileStatus);
        }
    }
}

2.2创建目录

@Test
public void mkdir() throws IllegalArgumentException, IOException {
    //3.在根目录下创建目录
    boolean mkdirs = fileSystem.mkdirs(new Path("/myhdfs"));//用Java操作hdfs创建的文件夹-目录
    System.out.println(mkdirs);
}

2.3删除目录

@Test
public void delete() throws IllegalArgumentException, IOException {
    //4.删除目录
    boolean delete = fileSystem.delete(new Path("/myhdfs"), true);
    System.out.println(delete);
}

@Test
public void deleteNotExist() throws IllegalArgumentException, IOException {
    //删除不存在的目录或文件会返回false
    boolean deleteNotExist = fileSystem.delete(new Path("/myhdfs1"), true);
    System.out.println(deleteNotExist);
}

2.4上传文件

@Test
public void uploadData() throws IllegalArgumentException, IOException {
    //5.上传文件-将Windows下hosts文件上传
    FSDataOutputStream create = fileSystem.create(new Path("/hosts1"));
    FileInputStream fsIn = new FileInputStream("C:\\Windows\\System32\\drivers\\etc\\hosts");
    IOUtils.copyBytes(fsIn, create, conf, true);
}

2.5读取文件

@Test
public void readFile() throws IOException {
    //6.读取文件
    FSDataInputStream open = fileSystem.open(new Path("/hosts1"));
    IOUtils.copyBytes(open, System.out, conf, true);
}

2.6按行读取文件

@Test
public void readFile2() throws IllegalArgumentException, IOException {
    //7.按行读取文件
    FSDataInputStream fsin = fileSystem.open(new Path("/input/test1.txt"));
    byte RL = '\r';//回车
    byte NL = '\n';//换行
    byte[] buffer = new byte[1024*64];
    int length = fsin.read(buffer);
    int start_pos = 0;
    int pos = 0;
    String str = null;
    for (pos = 0;pos < length; pos++) {
        //当为回车或者换行符时
        if (buffer[pos] == RL || buffer[pos] == NL) {
            //根据起始坐标start_pos和当前坐标生成字符串
            str = new String(buffer, start_pos, pos);
            System.out.println(str);

            pos++;//当前坐标加1
            start_pos = pos;//将新的位置赋给起始坐标
        }
    }
}

2.7删除文件

@Test
public void deleteFile() throws IllegalArgumentException, IOException {
    boolean deleteFile = fileSystem.delete(new Path("/hosts1"), true);//删除文件
    System.out.println(deleteFile);
}