HDFS基本操作

3.2 通过API操作HDFS

3.2.1 HDFS获取文件系统

1)详细代码

/**
   * 打印本地hadoop地址值
   * IO的方式写代码
   */
  @Test
  public void intiHDFS() throws IOException {
      //F2 可以快速的定位错误
      // alt + enter自动找错误
      //1.创建配信信息对象 ctrl + alt + v  后推前  ctrl + shitl + enter 补全
      Configuration conf = new Configuration();

      //2.获取文件系统
      FileSystem fs = FileSystem.get(conf);

      //3.打印文件系统
      System.out.println(fs.toString());
  }

3.2.2 HDFS文件上传

/**
    * 上传代码
    * 注意：如果上传的内容大于128MB,则是2块
    */
   @Test
   public void putFileToHDFS() throws Exception {
       //注：import org.apache.hadoop.conf.Configuration;
       //ctrl + alt + v 推动出对象
       //1.创建配置信息对象
       Configuration conf = new Configuration();

       //2.设置部分参数
       conf.set("dfs.replication","2");

       //3.找到HDFS的地址
       FileSystem fs = FileSystem.get(new URI("hdfs://bigdata111:9000"), conf, "root");

       //4.上传本地Windows文件的路径
       Path src = new Path("D:\\hadoop-2.7.2.rar");

       //5.要上传到HDFS的路径
       Path dst = new Path("hdfs://bigdata111:9000/");

       //6.以拷贝的方式上传，从src -> dst
       fs.copyFromLocalFile(src,dst);

       //7.关闭
       fs.close();

       System.out.println("上传成功");
}

3.2.3 HDFS文件下载

/**
    * hadoop fs -get /HDFS文件系统
    * @throws Exception
    */
   @Test
   public void getFileFromHDFS() throws Exception {
       //1.创建配置信息对象  Configuration:配置
       Configuration conf = new Configuration();

       //2.找到文件系统
       //final URI uri     ：HDFS地址
       //final Configuration conf：配置信息
       // String user ：Linux用户名
       FileSystem fs = FileSystem.get(new URI("hdfs://bigdata111:9000"), conf, "root");

       //3.下载文件
       //boolean delSrc:是否将原文件删除
       //Path src ：要下载的路径
       //Path dst ：要下载到哪
       //boolean useRawLocalFileSystem ：是否校验文件
       fs.copyToLocalFile(false,new Path("hdfs://bigdata111:9000/README.txt"),
               new Path("F:\\date\\README.txt"),true);

       //4.关闭fs
       //alt + enter 找错误
       //ctrl + alt + o  可以快速的去除没有用的导包
       fs.close();
       System.out.println("下载成功");
   }

3.2.4 HDFS目录创建

/**
  * hadoop fs -mkdir /xinshou
  */
 @Test
 public void mkmdirHDFS() throws Exception {
     //1.创新配置信息对象
     Configuration configuration = new Configuration();

     //2.链接文件系统
     //final URI uri  地址
     //final Configuration conf  配置
     //String user   Linux用户
     FileSystem fs = FileSystem.get(new URI("hdfs://bigdata111:9000"), configuration, "root");

     //3.创建目录
     fs.mkdirs(new Path("hdfs://bigdata111:9000/Good/Goog/Study"));

     //4.关闭
     fs.close();
     System.out.println("创建文件夹成功");
 }

3.2.5 HDFS文件夹删除

/**
    * hadoop fs -rm -r /文件
    */
   @Test
   public void deleteHDFS() throws Exception {
       //1.创建配置对象
       Configuration conf = new Configuration();

       //2.链接文件系统
       //final URI uri, final Configuration conf, String user
       //final URI uri  地址
       //final Configuration conf  配置
       //String user   Linux用户
       FileSystem fs = FileSystem.get(new URI("hdfs://bigdata111:9000"), conf, "root");

       //3.删除文件
       //Path var1   : HDFS地址
       //boolean var2 : 是否递归删除
       fs.delete(new Path("hdfs://bigdata111:9000/a"),false);

       //4.关闭
       fs.close();
       System.out.println("删除成功啦");
   }

3.2.6 HDFS文件名更改

````

@Test public void renameAtHDFS() throws Exception{ // 1 创建配置信息对象 Configuration configuration = new Configuration();

FileSystem fs = FileSystem.get(new URI("hdfs://bigdata111:9000"),configuration, "itstar");
	
    //2 重命名文件或文件夹
    fs.rename(new Path("hdfs://bigdata111:9000/user/itstar/hello.txt"), new Path("hdfs://bigdata111:9000/user/itstar/hellonihao.txt"));
fs.close();
}

````

3.2.7 HDFS文件详情查看

/**
     * 查看【文件】名称、权限等
     */
    @Test
    public void readListFiles() throws Exception {
        //1.创建配置对象
        Configuration conf = new Configuration();

        //2.链接文件系统
        FileSystem fs = FileSystem.get(new URI("hdfs://bigdata111:9000"), conf, "root");

        //3.迭代器
//List the statuses and block locations of the files in the given path. If the path is a directory, if recursive is false, returns files in the directory; if recursive is true, return files in the subtree rooted at the path. If the path is a file, return the file's status and block locations.

        RemoteIterator<LocatedFileStatus> listFiles = fs.listFiles(new Path("/"), true);

        //4.遍历迭代器
        while (listFiles.hasNext()){
            //一个一个出
            LocatedFileStatus fileStatus = listFiles.next();

            //名字
            System.out.println("文件名：" + fileStatus.getPath().getName());
            //块大小
            System.out.println("大小：" + fileStatus.getBlockSize());
            //权限
            System.out.println("权限：" + fileStatus.getPermission());
            System.out.println(fileStatus.getLen());


            BlockLocation[] locations = fileStatus.getBlockLocations();

            for (BlockLocation bl:locations){
                System.out.println("block-offset:" + bl.getOffset());
                String[] hosts = bl.getHosts();
                for (String host:hosts){
                    System.out.println(host);
                }
            }

            System.out.println("------------------华丽的分割线----------------");
        }

3.2.8 HDFS文件和文件夹判断

/**
     * 判断是否是个文件还是目录，然后打印
     * @throws Exception
     */
    @Test
    public void judge() throws Exception {
        //1.创建配置文件信息
        Configuration conf = new Configuration();

        //2.获取文件系统
        FileSystem fs = FileSystem.get(new URI("hdfs://bigdata111:9000"), conf, "root");

        //3.遍历所有的文件
//List the statuses of the files/directories in the given path if the path is a directory.
   FileStatus[] liststatus = fs.listStatus(new Path("/"));
        for(FileStatus status :liststatus)
        {
            //判断是否是文件
            if (status.isFile()){
                //ctrl + d:复制一行
                //ctrl + x 是剪切一行，可以用来当作是删除一行
                System.out.println("文件:" + status.getPath().getName());
            } else {
                System.out.println("目录:" + status.getPath().getName());
            }
        }
    }

3.3 通过IO流操作HDFS

3.3.1 HDFS文件上传

/**
   * IO流方式上传
   *
   * @throws URISyntaxException
   * @throws FileNotFoundException
   * @throws InterruptedException
   */
  @Test
  public void putFileToHDFSIO() throws URISyntaxException, IOException, InterruptedException {
      //1.创建配置文件信息
      Configuration conf = new Configuration();

      //2.获取文件系统
      FileSystem fs = FileSystem.get(new URI("hdfs://bigdata111:9000"), conf, "root");

      //3.创建输入流
      FileInputStream fis = new FileInputStream(new File("F:\\date\\Sogou.txt"));

      //4.输出路径
      //注意：不能/plus  记得后边写个名 比如：/plus/Sogou.txt
      Path writePath = new Path("hdfs://bigdata111:9000/plus/Sogou.txt");
      FSDataOutputStream fos = fs.create(writePath);

      //5.流对接
      //InputStream in    输入
      //OutputStream out  输出
      //int buffSize      缓冲区
      //boolean close     是否关闭流
      try {
          IOUtils.copyBytes(fis,fos,4 * 1024,false);
      } catch (IOException e) {
          e.printStackTrace();
      }finally {
          IOUtils.closeStream(fos);
          IOUtils.closeStream(fis);
fs.close();

          System.out.println("上传成功啦");
      }
  }

3.3.2 HDFS文件下载

/**
    * IO读取HDFS到控制台
    *
    * @throws URISyntaxException
    * @throws IOException
    * @throws InterruptedException
    */
   @Test
   public void getFileToHDFSIO() throws URISyntaxException, IOException, InterruptedException {
       //1.创建配置文件信息
       Configuration conf = new Configuration();

       //2.获取文件系统
       FileSystem fs = FileSystem.get(new URI("hdfs://bigdata111:9000"), conf, "root");

       //3.读取路径
       Path readPath = new Path("hdfs://bigdata111:9000/plus/Sogou.txt");

       //4.输入
       FSDataInputStream fis = fs.open(readPath);

       //5.输出到控制台
       //InputStream in    输入
       //OutputStream out  输出
       //int buffSize      缓冲区
       //boolean close     是否关闭流
       IOUtils.copyBytes(fis,System.out,4 * 1024 ,true);
   }

3.3.3 定位文件读取

1)下载第一块

/**
    * IO读取第一块的内容
    *
    * @throws Exception
    */
   @Test
   public void  readFlieSeek1() throws Exception {
       //1.创建配置文件信息
       Configuration conf = new Configuration();

       //2.获取文件系统
       FileSystem fs = FileSystem.get(new URI("hdfs://bigdata111:9000"), conf, "root");

       //3.输入
       Path path = new Path("hdfs://bigdata111:9000/plus/hadoop-2.7.2.rar");
       FSDataInputStream fis = fs.open(path);

       //4.输出
       FileOutputStream fos = new FileOutputStream("F:\\date\\readFileSeek\\A1");

       //5.流对接
       byte[] buf = new byte[1024];
       for (int i = 0; i < 128 * 1024; i++) {
           fis.read(buf);
           fos.write(buf);
       }

       //6.关闭流
       IOUtils.closeStream(fos);
       IOUtils.closeStream(fis);
   }

2)下载第二块

/**
     * IO读取第二块的内容
     *
     * @throws Exception
     */
    @Test
    public void readFlieSeek2() throws Exception {
        //1.创建配置文件信息
        Configuration conf = new Configuration();

        //2.获取文件系统
        FileSystem fs = FileSystem.get(new URI("hdfs://bigdata111:9000"), conf, "root");

        //3.输入
        Path path = new Path("hdfs://bigdata111:9000/plus/hadoop-2.7.2.rar");
        FSDataInputStream fis = fs.open(path);

        //4.输出
        FileOutputStream fos = new FileOutputStream("F:\\date\\readFileSeek\\A2");

        //5.定位偏移量/offset/游标/读取进度 (目的：找到第一块的尾巴，第二块的开头)
        fis.seek(128 * 1024 * 1024);

        //6.流对接
        IOUtils.copyBytes(fis, fos, 1024);

        //7.关闭流
        IOUtils.closeStream(fos);
        IOUtils.closeStream(fis);
    }

3)合并文件

在window命令窗口中执行

type A2 » A1 然后更改后缀为rar即可