hadoop-hdfs的API开发

在前一博客中我们搭建了hdfs的开发环境，接下来我们要在idea中进行相应的代码开发
1、引入依赖包

<dependency>
      <groupId>junit</groupId>
      <artifactId>junit</artifactId>
      <version>4.11</version>
      <scope>test</scope>
    </dependency>

    <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-common -->
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-common</artifactId>
      <version>2.6.5</version>
    </dependency>
    <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-hdfs -->
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-hdfs</artifactId>
      <version>2.6.5</version>
    </dependency>
    <dependency>
      <groupId>junit</groupId>
      <artifactId>junit</artifactId>
      <version>4.12</version>
      <scope>compile</scope>
    </dependency>

因为需要进行测试，所以我们使用junit进行单元测试，测试每一个功能模块是否好用
接下来我们开始代码的编写，以下代码包括hdfs的增删改查、上传、下载以及block信息查询等功能

package com.msb.hdfs;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.io.*;
import java.net.URI;


public class TestHdfs {

    public Configuration conf = null;
    public FileSystem fs = null;

    @Before
    public void conn() throws IOException, InterruptedException {

        conf = new Configuration(true);
        //取环境变量，创建链接
        //fs = FileSystem.get(conf);
        //直接连接集群
        fs = FileSystem.get(URI.create("hdfs://node01:9000/"), conf, "root");
    }

    //创建，删除，查询
    @Test
    public void mkdir() throws IOException {
        Path dir = new Path("/Test");
        if (fs.exists(dir)) {
            fs.delete(dir, true);
        }
        //创建目录
        fs.mkdirs(dir);
    }

    //上传文件
    @Test
    public void upload() throws Exception {
        //创建输入
        BufferedInputStream input = new BufferedInputStream(new FileInputStream(new File("./data/hello.txt")));
        //创建输出
        Path outfile = new Path("/Test/out.txt");
        FSDataOutputStream output = fs.create(outfile);
        IOUtils.copyBytes(input,output,conf,true);
    }
    //下载文件
    @Test
    public void getFileHdfs() throws IOException {
        //定义文件Hdfs下载的路径
        Path output = new Path("/Test/out.txt");
        //定义下载到win中文件路径
        Path input = new Path("D:/student.txt");
        //下载
        fs.copyToLocalFile(false, output, input, true);
        //关闭
        fs.close();
        System.out.println("plus下载成功");
    }

    @Test
    public void blocks() throws Exception {
        Path path = new Path("/Test/out.txt");
        FileStatus status = fs.getFileStatus(path);
        BlockLocation[] blks = fs.getFileBlockLocations(status, 0, status.getLen());
        for (BlockLocation blk:blks) {
            System.out.println(blk);
        }


    }

    @After
    public void close() throws IOException {
        fs.close();
    }
}

执行，然后再我们的hdfs集群中通过

web页面或者命令hdfs dfs -ls /

进行相应的查看

标注：
http://archive.apache.org/dist/
文档下载地址