大数据第三天————IDEA下自定义 HDFS API基本操作(封装hdfs基本操作)

最近研究了一波HDFS基本操作，如果java代码写的话，会非常麻烦，所以我封装了一下一些主要的HDFS操作。而且正常使用命令行操作，是看不到成功与否，所以在封装类里面增加了结果打印情况，发现查看hadoop的基本操作。同时还增加了批量操作的封装。

目录

1.封装函数的接口

 2.封装函数的实现

3.实例应用

4.运行截图

1.封装函数的接口

import java.io.IOException;
import java.net.URISyntaxException;

/**
 * Title:    IHDFS
 * Description:    hdfs操作接口
 *
 * @author dtt
 * @data 2018-09-05 16:46
 **/

public interface IHDFS {
//    显示文件
    void cat(String folder) throws IOException, URISyntaxException;
//    删除文件
    void rm(String folder) throws IOException, URISyntaxException;
//    删除多个文件
    void rms(String[] folders) throws IOException, URISyntaxException;
//    下载文件
    void get(String remote,  String local) throws IllegalArgumentException, IOException, URISyntaxException;
//    下载多个文件
    void gets(String[] remotes,  String[] locals) throws IllegalArgumentException, IOException, URISyntaxException;
//    上传文件
    void put(String local, String remote) throws IOException, URISyntaxException;
//    上传多个文件
    void puts(String[] locals, String[] remotes) throws IOException, URISyntaxException;
//    递归列出所有目录
    void lsr(String folder) throws IOException, URISyntaxException;
//    列出所有目录
    void ls(String folder) throws IOException, URISyntaxException;
//    删除目录
    void rmr(String folder) throws IOException, URISyntaxException;
//    删除多个目录
    void rmrs(String[] folder) throws IOException, URISyntaxException;
//    创建目录
    void mkdir(String folder) throws IOException, URISyntaxException;
//    创建多个目录
    void mkdirs(String[] folders) throws IOException, URISyntaxException;
//    重命名文件
    void rename(String old, String now) throws IOException, URISyntaxException;

//    待更新...
}

 2.封装函数的实现

package cn.tedu.utils;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;


/**
 * Title:    HDFS
 * Description:    工具分装
 *
 * @author dtt
 * @data 2018-09-05 15:38
 **/

public class HDFS implements IHDFS{
    private String url;
    private Configuration conf;
    private FileSystem fs;
    public HDFS(String url){
        this.url = url;
        this.conf = new Configuration();
    }

//    显示文件
    public void cat(String folder) throws IOException, URISyntaxException{
        fs = FileSystem.get(new URI(url),conf);
        System.out.println("cat: " + folder);
        Path path = new Path(folder);
        FSDataInputStream fsdis = fs.open(path);
        IOUtils.copyBytes(fsdis,System.out,4096,false);
        IOUtils.closeStream(fsdis);
        fs.close();
    }

//    删除文件
    public void rm(String folder) throws IOException, URISyntaxException{
        fs = FileSystem.get(new URI(url),conf);
        Path path = new Path(folder);
        if(fs.deleteOnExit(path)){
            fs.delete(path,false);
            System.out.println("delete:" + folder);
        }else{
            System.out.println("The file is not exist!");
        }
        fs.close();
    }

//    删除多个文件
    public void rms(String[] folders) throws IOException, URISyntaxException{
        fs = FileSystem.get(new URI(url),conf);
        for (String folder : folders) {
            Path path = new Path(folder);
            if(fs.deleteOnExit(path)){
                fs.delete(path,false);
                System.out.println("delete:" + folder);
            }else{
                System.out.println("The file is not exist!");
            }
        }
        fs.close();
    }

//    下载文件
    public void get(String remote,  String local) throws IllegalArgumentException, IOException, URISyntaxException {
        // 建立联系
        fs = FileSystem.get(new URI(url),conf);
        fs.copyToLocalFile(new Path(remote), new Path(local));
        System.out.println("Get From :   " + remote  + "   To :" + local);
        fs.close();
    }

//    下载多个文件
    public void gets(String[] remotes,  String[] locals) throws IllegalArgumentException, IOException, URISyntaxException {
        // 建立联系
        fs = FileSystem.get(new URI(url),conf);
        int len = remotes.length;
        for(int i=0;i<len;i++){
            fs.copyToLocalFile(new Path(remotes[i]), new Path(locals[i]));
            System.out.println("Get From :   " + remotes[i]  + "   To :" + locals[i]);
        }
        fs.close();
    }

//    上传文件
    public void put(String local, String remote) throws IOException, URISyntaxException {
        // 建立联系
        fs = FileSystem.get(new URI(url),conf);
        fs.copyFromLocalFile(new Path(local), new Path(remote));
        System.out.println("Put :" + local  + "   To : " + remote);
        fs.close();
    }

//    上传多个文件
    public void puts(String[] locals, String[] remotes) throws IOException, URISyntaxException {
        // 建立联系
        fs = FileSystem.get(new URI(url),conf);
        int len = locals.length;
        for(int i=0;i<len;i++){
            fs.copyFromLocalFile(new Path(locals[i]), new Path(remotes[i]));
            System.out.println("Put :" + locals[i]  + "   To : " + remotes[i]);
        }
        fs.close();
    }

//    递归列出全部目录
    public void lsr(String folder) throws IOException, URISyntaxException {
        //与hdfs建立联系
        fs = FileSystem.get(new URI(url),conf);
        Path path = new Path(folder);
        //得到该目录下的全部文件
        FileStatus[] fileList = fs.listStatus(path);
        for (FileStatus f : fileList) {
            System.out.printf("name: %s   |   folder: %s  |   size: %d\n", f.getPath(),  f.isDirectory() , f.getLen());
            try{
                FileStatus[] fileListR = fs.listStatus(f.getPath());
                for(FileStatus fr:fileListR){
                    System.out.printf("name: %s   |   folder: %s  |   size: %d\n", fr.getPath(),  fr.isDirectory() , fr.getLen());
                }
            }finally{
                continue;
            }
        }
        fs.close();
    }

//    列出全部目录
    public void ls(String folder) throws IOException, URISyntaxException {
        //与hdfs建立联系
        fs = FileSystem.get(new URI(url),conf);
        Path path = new Path(folder);
        //得到该目录下的全部文件
        FileStatus[] fileList = fs.listStatus(path);
        for (FileStatus f : fileList) {
            System.out.printf("name: %s   |   folder: %s  |   size: %d\n", f.getPath(),  f.isDirectory() , f.getLen());
        }
        fs.close();
    }

//    删除目录
    public void rmr(String folder) throws IOException, URISyntaxException {
        //与hdfs建立联系
        fs = FileSystem.get(new URI(url),conf);
        Path path = new Path(folder);
        fs.delete(path,true);
        System.out.println("delete:" + folder);
        fs.close();
    }

//    删除多个目录
    public void rmrs(String[] folders) throws IOException, URISyntaxException {
        //与hdfs建立联系
        fs = FileSystem.get(new URI(url),conf);
        for (String folder : folders) {
            Path path = new Path(folder);
            fs.delete(path,true);
            System.out.println("delete:" + folder);
        }
        fs.close();
    }

//    创建目录
    public void mkdir(String folder) throws IOException, URISyntaxException {
        //与hdfs建立联系
        fs = FileSystem.get(new URI(url),conf);
        Path path = new Path(folder);
        if (!fs.exists(path)) {
            fs.mkdirs(path);
            System.out.println("Create: " + folder);
        }else{
            System.out.println("it is have exist:" + folder);
        }
        fs.close();
    }

//    创建多个目录
    public void mkdirs(String[] folders) throws IOException, URISyntaxException {
        //与hdfs建立联系
        fs = FileSystem.get(new URI(url),conf);
        for (String folder : folders) {
            Path path = new Path(folder);
            if (!fs.exists(path)) {
                fs.mkdirs(path);
                System.out.println("Create: " + folder);
            }else{
                System.out.println("it is have exist:" + folder);
            }
        }
        fs.close();
    }

//    判断文件是否存在
    private void isExist(String folder) throws IOException, URISyntaxException {
        //与hdfs建立联系
        fs = FileSystem.get(new URI(url),conf);
        Path path = new Path(folder);
        if(fs.exists(path)){
            System.out.println("it is have exist:" + folder);
        }else{
            System.out.println("it is not exist:" + folder);
        }
        fs.close();
    }
//    重命名文件
    public void rename(String old, String now) throws IOException, URISyntaxException{
        //与hdfs建立联系
        fs = FileSystem.get(new URI(url),conf);
        Path path = new Path(old);
        Path path1 = new Path(now);
        if(fs.exists(path)){
            fs.rename(path,path1);
            System.out.println(old + "renamed to " + now);
        } else{
            System.out.println("it is not exist:" + old);
        }
    }

}

3.实例应用

import cn.tedu.utils.HDFS;
/**
 * Title:    test0905
 * Description:    0905作业
 *
 * @author dtt
 * @data 2018-09-05 19:56
 **/

public class test0905 {
    // 添加自己的IP
    private static String myurl = "hdfs://自己的IP:9000";
    private static String localdir = "C:/Users/DemonTT/Desktop";
    public static void main(String[] args) throws Exception{
        HDFS hdfs = new HDFS(myurl);
        // 1.在hdfs创建目录teacher
        hdfs.mkdir("/teacher");

        // 2.在teacher目录下上传文件score.txt
        hdfs.put(localdir+"/score.txt","/teacher");

        // 3.在hdfs创建目录student，并在student目录下创建新目录Tom、LiMing、Jerry.
        hdfs.mkdirs(new String[]{"/student","/student/Tom","/student/LiMing","/student/Jerry"});

        // 4.在Tom目录下上传information.txt,同时上传到LiMing、Jerry目录下。
        hdfs.puts(new String[]{localdir+"/information.txt",localdir+"/information.txt",localdir+"/information.txt"},new String[]{"/student/Tom","/student/LiMing","/student/Jerry"});

        // 5.将student重命名为MyStudent。
        hdfs.rename("/student","/MyStudent");

        // 6.将Tom下的information.txt下载到D:/tom目录中
//        hdfs.get("/student/Tom/information.txt",localdir+"/down_information.txt");

        // 7.将teacher下的score.txt也下载到此目录
	// 合并6、7命令
        hdfs.gets(new String[]{"/MyStudent/Tom/information.txt","/teacher/score.txt"},new String[]{localdir+"/down_information.txt",localdir+"/down_score.txt"});

        // 8.删除hdfs中的Tom、LiMing目录
        hdfs.rmrs(new String[]{"/MyStudent/Tom","/MyStudent/LiMing"});
//    如果想要查看文件，请打开下面的注释
        /*hdfs.ls("/teacher");

        hdfs.ls("/MyStudent");*/

    }
}

4.运行截图