大数据之Hadoop（HDFS Java客户端操作）

最新推荐文章于 2024-05-08 08:23:21 发布

袖清暮雨

最新推荐文章于 2024-05-08 08:23:21 发布

阅读量202

点赞数 3

文章标签：大数据 hadoop hdfs

本文链接：https://blog.csdn.net/qq_73339471/article/details/137637162

版权

本文详细介绍了HDFS类的设计，包括属性配置、文件系统交互、文件/目录操作（如存在性检查、类型判断、删除、创建、上传/下载）以及HDFSTest示例。重点展示了如何使用HDFS进行文件系统管理和数据传输。

摘要由CSDN通过智能技术生成

"HDFS类"的设计

属性的设计

//属性设计
//属性
private Configuration config;//配置
private FileSystem fileSystem;//用于与HDFS获取交互
//属性初始化
public HDFS(String url, Map<String,String> configItems) throws IOException, InterruptedException {
    //配置
    config = new Configuration();
    //对象不为空【Objects.nonNull(configItems)】及 值不为0【!configItems.isEmpty()】
    if (Objects.nonNull(configItems) && !configItems.isEmpty()){
        configItems.entrySet().forEach(e->config.set(e.getKey(),e.getValue()));
    }
    //连接HDFS
    fileSystem = FileSystem.get(URI.create(url),config,"root");
}

功能的设置

1.是否存在

//1.是否存在
public boolean exists(String path) throws IOException {
    return fileSystem.exists(new Path(path));
}

2.类型判断

//类型枚举
public enum TYPE{
    DIRECTORY,FILE,NOT_EXIST
}

//2.类型
public TYPE type(String path) throws IOException {
    if (!exists(path)){
        return TYPE.NOT_EXIST;
    }
    Path pt=new Path(path);
    if (fileSystem.getFileStatus(pt).isFile()){
        return TYPE.FILE;
    }
    return TYPE.DIRECTORY;
}

3.删除

//3.删除
public void deleteIfExist(String path) throws IOException {
    Path ps = new Path(path);
    switch (type(path)){
        case FILE:
            fileSystem.delete(ps,false);//不递归
            break;
        case DIRECTORY:
            fileSystem.delete(ps,true);//递归
            break;
    }
}

4.创建目录

//4.创建目录
//针对目录，不存在则递归创建
public void mkdirIfNotExist(String path) throws IOException {
    if (!exists(path)){
        fileSystem.mkdirs(new Path(path));
    }
}

5.上传

//5.将本地资源(文件 or 目录)【上传】到 HDFS上
//currDir:当前目录
//destDir:目标目录
public void uploadRecursive(File currDir,Path destDir,int currDepth,int maxDepth) throws IOException {
    if (maxDepth<=0 && currDepth>maxDepth){
        return;
    }
    for (File res : currDir.listFiles()) {
        if (res.isDirectory()){
            uploadRecursive(res,destDir,currDepth+1,maxDepth);
        }else{
            fileSystem.copyFromLocalFile(new Path(res.getPath()),destDir);
        }
    }
}

public void upload(String localPath,String dfsPath,boolean recursive,int...maxDepth) throws IOException {
        //判断
        File res = new File(localPath);//上传对象
        if (!res.exists()){
            throw new IOException("you can't upload the localPath that is not exist");
        }
        if (type(dfsPath)!= TYPE.DIRECTORY){
            throw new IOException("you can't upload any resource to a file that it is directory");
        }
        Path local=new Path(localPath);
        Path remote=new Path(dfsPath);
        if (res.isFile()){
            fileSystem.copyFromLocalFile(local,remote);
        }else if (recursive){
            int max = maxDepth[0];
            max = max<0?0:max;
            uploadRecursive(res,remote,1,max);
        }else {
            throw new IOException("you can't upload a directory when parameter recursive was set false");
        }

    }

6.下载

//6.从HDFS上【下载】至本地
public void download(String dfsPath,String localDir) throws IOException {
    //判断
    if (exists(dfsPath)){
        File local;
        if ((local=new File(localDir)).exists()){
            Path remote = new Path(dfsPath);
            Path localPath = new Path(localDir);
            fileSystem.copyToLocalFile(remote,localPath);
        }else {
            throw new IOException("you can't download a resource to local file");
        }
    }else {
        throw new IOException("you can't download a remote resource that is not exist");
    }
}

7.关闭

//7.关闭
public void close(){
    try {
        fileSystem.close();
    } catch (IOException e) {
        e.printStackTrace();
    }
}

"HDFS类"实际操作

代码展示

public class HDFSTest {
    public static void main(String[] args) {
        Map<String,String> config = new HashMap<>();
        config.put("dfs.replication","3");//每个HDFS数据块的副本数量
        config.put("dfs.blocksize","128M");//HDFS块的大小
        final String URL = "hdfs://single:9000";

        HDFS hdfs = null;
        try {
            hdfs = new HDFS(URL,config);

            //以下内容为操作内容：
            //----------------------------------------------------------------
            //创建目录
            hdfs.mkdirIfNotExist("/hadoop/data/h");
            ...
            //----------------------------------------------------------------

        } catch (IOException e) {
            e.printStackTrace();
        } catch (InterruptedException e) {
            e.printStackTrace();
        }finally {
            if (Objects.nonNull(hdfs)){
                hdfs.close();
            }
        }
    }
}