HDFS：Idea开发Client

写的不到位的地方，欢迎评论指出不足之处

前言：

        前提是服务器端部署正常

开发工具：

        IntelliJ IDEA、Maven

---

1、配置 pom.xml

<dependency>
  <groupId>junit</groupId>
  <artifactId>junit</artifactId>
  <version>4.11</version>
  <scope>compile</scope>
</dependency>

<dependency>
  <groupId>org.apache.hadoop</groupId>
  <artifactId>hadoop-common</artifactId>
  <version>2.6.5</version>
</dependency>

<dependency>
  <groupId>org.apache.hadoop</groupId>
  <artifactId>hadoop-hdfs</artifactId>
  <version>2.6.5</version>
  <scope>test</scope>
</dependency>

2、将 Hadoop 配置文件 core-site.xml、hdfs-site.xml 复制到项目的 resources 下

3、代码

package com.my.hadoop.hdfs;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.io.*;
import java.net.URI;

public class TestHDFS {

    public Configuration conf = null;
    public FileSystem fs = null;


    @Before
    public void conn() throws Exception {
        conf = new Configuration(true);
//        fs = FileSystem.get(conf);
//        <property>
//          <name>fs.defaultFS</name>
//          <value>hdfs://myHA</value>
//        </property>
//      环境变量 HADOOP_USER_NAME root
        fs = FileSystem.get(URI.create("hdfs://myHA"), conf, "root");

    }

    @Test
    public void mkdir() throws Exception {
        Path dir = new Path("/data/root");
        if (fs.exists(dir)) {
            fs.delete(dir, true);
        }
        fs.mkdirs(dir);
    }

    @Test
    public void upload() throws Exception {
        BufferedInputStream input = new BufferedInputStream(new FileInputStream(new File("./data/hello.txt")));
        Path outfile = new Path("/data/root/hello.txt");
        FSDataOutputStream output = fs.create(outfile);
        IOUtils.copyBytes(input, output, conf, true);
    }

    @Test
    public void download() throws Exception {
        Path infile = new Path("/data/root/hello.txt");
        // 面向文件打开输入流，无论怎么读都是从文件开始读取
        FSDataInputStream input = fs.open(infile);
        BufferedOutputStream output = new BufferedOutputStream(new FileOutputStream("./data/hello2.txt"));
        IOUtils.copyBytes(input, output, conf, true);

    }

    /**
     * 0,1048576,four,two
     * 1048576,1048576,three,two
     * 起始字节位置，块大小1M，分别在哪个位置
     *
     * @throws Exception
     */
    @Test
    public void blocksLocation() throws Exception {
        Path file = new Path("/data/root/hello.txt");
        FileStatus fss = fs.getFileStatus(file);
        BlockLocation[] block = fs.getFileBlockLocations(fss, 0, fss.getLen());
        // 用户和程序读取的是文件这个级别，并不知道有块的概念
        for (Object i : block) {
            System.out.println(i);
        }
    }

    @Test
    public void blocks() throws Exception {
        Path infile = new Path("/data/root/hello.txt");
        FSDataInputStream input = fs.open(infile);
        // 查看第一块的最后2行内容：tail -n 2 blk_1073741826
        // 查看第二块的开头2行内容：head -n 2 blk_1073741827
        // input.seek(1048576);
        // 计算向数据移动后，期望的是分治，只读取自己关心（通过seek实现）
        // 同时，具有距离的概念（优先和本地的DataNode获取数据，框架的默认机制）
        for (int i = 0; i <= 10; i++) {
            System.out.println((char) input.readByte());
        }
    }

    @After
    public void close() throws Exception {
        fs.close();
    }

}