Java操作HDFS

Java操作HDFS

maven仓库【阿里仓库--支持chd的下载】

• 配置文件

   

  <mirror>
      <id>nexus-aliyun</id>
      <mirrorOf>*,!cloudera</mirrorOf>
      <name>Nexus aliyun</name>                     
      <url>
        http://maven.aliyun.com/nexus/content/groups/public
      </url>
  </mirror>
  

pom文件

• 配置文件

   

  <?xml version="1.0" encoding="UTF-8"?>
  <project xmlns="http://maven.apache.org/POM/4.0.0"
           xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
           xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
      <modelVersion>4.0.0</modelVersion>
  
      <!--根据自己创建的项目写start-->
      <groupId>com.peng</groupId>
      <artifactId>hdfstest</artifactId>
      <version>1.0-SNAPSHOT</version>
      <!--根据自己创建的项目写end-->
  
      <repositories>
          <repository>
              <id>cloudera</id>
              <url>https://repository.cloudera.com/content/repositories/releases/</url>
          </repository>
      </repositories>
  
      <dependencies>
          <dependency>
              <groupId>org.apache.hadoop</groupId>
              <artifactId>hadoop-client</artifactId>
              <version>2.6.0-cdh5.7.0</version>
          </dependency>
  
          <dependency>
              <groupId>junit</groupId>
              <artifactId>junit</artifactId>
              <version>4.10</version>
              <scope>test</scope>
          </dependency>
  
      </dependencies>
  
  </project>
  

单元测试文件

• HdfsTest.java文件

   

  package com.peng;
  
  import org.apache.hadoop.conf.Configuration;
  import org.apache.hadoop.fs.*;
  import org.apache.hadoop.io.IOUtils;
  import org.apache.hadoop.util.Progressable;
  import org.junit.After;
  import org.junit.Before;
  import org.junit.Test;
  
  import java.io.BufferedInputStream;
  import java.io.File;
  import java.io.FileInputStream;
  import java.io.InputStream;
  import java.net.URI;
  
  public class HdfsTest {
      //文件系统
      private FileSystem fileSystem = null;
      //配置文件
      private Configuration configuration = null;
      //虚拟机的HDFS的访问URI
      private static final String HDFS_URI = "hdfs://hadoop01/8020";
  
      //测试--创建目录
      @Test
      public void test1() throws Exception {
          boolean is_mkdirs = fileSystem.mkdirs(new Path("/hdfs_test"));
          if (is_mkdirs) {
              System.out.println("创建目录成功！");
          } else {
              System.out.println("创建目录失败");
          }
      }
  
      //测试--创建txt文件
      @Test
      public void test2() throws Exception {
          FSDataOutputStream fsDataOutputStream = fileSystem.create(new Path("/hdfs_test/hello.txt"));
          fsDataOutputStream.write("hello,hdfs!".getBytes());
      }
  
      //测试--查看txt文件的内容
      @Test
      public void test3() throws Exception {
          FSDataInputStream open = fileSystem.open(new Path("/hdfs_test/hello.txt"));
          IOUtils.copyBytes(open, System.out, 1024);
      }
  
      //测试--重命名文件
      @Test
      public void test4() throws Exception {
          Path oldPath = new Path("/hdfs_test/hello.txt");
          Path newPath = new Path("/hdfs_test/hello_new.txt");
          boolean is_rename = fileSystem.rename(oldPath, newPath);
          if (is_rename) {
              System.out.println("重命名文件成功！");
          } else {
              System.out.println("重命名文件失败！");
          }
      }
  
      //测试--上传文件到hdfs
      @Test
      public void test5() throws Exception {
          Path windowsPath = new Path("C:\\Users\\kungfupeng\\Desktop\\lzh.pdf");
          Path linuxPath = new Path("/hdfs_test");
          fileSystem.copyFromLocalFile(windowsPath, linuxPath);
      }
  
  
      //测试--上传文件到hdfs--加进度条
      @Test
      public void test6() throws Exception {
          InputStream in = new BufferedInputStream(new FileInputStream(new File("C:\\Users\\kungfupeng\\Downloads\\test_upload.zip")));
          FSDataOutputStream fsDataOutputStream = fileSystem.create(new Path("/hdfs_test/test_upload.zip"), new Progressable() {
              public void progress() {
                  System.out.print("*");
              }
          });
  
          IOUtils.copyBytes(in, fsDataOutputStream, 4096);
      }
  
      //测试--下载hdfs文件
      @Test
      public void test7() throws Exception {
          Path linuxPath = new Path("/hdfs_test/lzh.pdf");
          Path windowsPath = new Path("C:\\Users\\kungfupeng\\Desktop\\lzh_upload_linux.pdf");
          fileSystem.copyToLocalFile(linuxPath, windowsPath);
      }
  
      //测试-删除文件
      @Test
      public void test8() throws Exception {
          boolean is_delete = fileSystem.delete(new Path("/hdfs_test/test_upload.zip"), true);//true/false是否是用递归进行删除
          if (is_delete) {
              System.out.println("删除文件成功！");
          } else {
              System.out.println("删除文件失败！");
          }
      }
  
      //查看某一个目录下的所有文件
      @Test
      public void test9() throws Exception {
          FileStatus[] fileStatuses = fileSystem.listStatus(new Path("/hdfs_test"));
          for (FileStatus fs : fileStatuses) {
              System.out.println("这里的副本系数为3---java方式上传的话，副本采用的是hadoop自己的副本系数");
              String isDir = fs.isDirectory() ? "文件夹" : "文件";
              short replication = fs.getReplication();
              long len = fs.getLen();
              String s = fs.getPath().toString();
              System.out.println("文件or文件夹？【" + isDir + "】\t副本【" + replication + "】\t长度【" + len + "】\t路径【" + s + "】");
          }
      }
  
      //开始前操作--初始化资源
      @Before
      public void setUp() throws Exception {
          System.out.println("=====================start========================");
          configuration = new Configuration();
          fileSystem = FileSystem.get(new URI(HDFS_URI), configuration, "root");
      }
  
      //结束后操作--释放资源
      @After
      public void clearTail() throws Exception {
          configuration = null;
          fileSystem = null;
          System.out.println("=====================end========================");
      }
  }
  

测试结果快速查看

• 浏览器
  • 地址：http://hadoop01:50070/explorer.html#/hdfs_test
  • 注：hadoop01是虚拟机的地址，自行变换
• 虚拟机
  • 命令
    1. hdfs dfs -ls /