HDFS基础文件操作实验

提示：文章写完后，目录可以自动生成，如何生成可参考右边的帮助文档

---

一、HDFS的基本操作

基于已经学习到的Hadoop API编程知识，自己动手实现一个简单的HDFS Shell程序，程序名称为HShell，要求能够支持以下功能：

1.使用HShell -cp 本地路径 HDFS路径，将文件从Linux本地文件系统拷贝到HDFS指定路径上。

2.使用HShell -rm 路径删除文件

3.使用HShell -rm -r 路径删除目录

4.使用HShell -cp -r 本地目录路径 HDFS路径，将目录从Linux本地拷贝到HDFS指定路径上。

5.使用HShell -list 路径显示某个文件的信息或者某个目录的信息

6.使用HShell -mv 路径 路径移动文件或者重命名文件

7.使用HShell -find 文件名 目录实现在目录下递归查找某个文件名的文件。

二、完整代码

package homework;
import java.util.Scanner;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;

import java.net.URI;
public class HShell {
    public static void main(String[] args){
        String choice;
        while(true) {
            Scanner sc = new Scanner(System.in);
            choice = sc.nextLine();
            if(choice.equals("exit")){
                System.out.println("HShell have exited");
                break;
            }
            String[]  shell=choice.split(" ");
            if(shell[2].equals("-r")){
                choice=shell[0]+" "+shell[1]+" "+shell[2];
            }else{
                choice=shell[0]+" "+shell[1];
            }
            switch (choice) {
                case "HShell -rm":
                    if(shell.length>4){
                        System.out.println("input error,please input again");
                    }else {
                        DeleteFile(shell[2]);
                    }
                    break;
                case "HShell -rm -r":
                    if(shell.length>5){
                        System.out.println("input error,please input again");
                    }else{
                        DeleteDir(shell[3]);
                    }
                    break;
                case "HShell -list":
                    if (shell.length>4){
                        System.out.println("input error,please input again");
                    }else {
                        ListFile(shell[2]);
                    }
                    break;
                case "HShell -mv":
                    if(shell.length>5){
                        System.out.println("input error,please input again");
                    }else {
                        Rename(shell[2],shell[3]);
                        break;
                    }
                case "HShell -cp":
                    if (shell.length>5){
                        System.out.println("input error,please input again");
                    }else {
                        UploadFile(shell[2],shell[3]);
                    }
                    break;
                case "HShell -cp -r":
                    if(shell.length>6){
                        System.out.println("input error,please input again");
                    }else {
                        UploadDir(shell[3],shell[4]);
                        break;
                    }
                default:
                    System.out.println("no such HShell code,please input again");
                    break;
                case "HShell -find":
                    if(shell.length>5){
                        System.out.println("input error,input again");
                        break;
                    }else {
                        Findfile(shell[2],shell[3]);
                    }
                    break;
            }
        }
    }
    public static void DeleteFile(String filename){
        try {
            //Scanner sc = new Scanner(System.in);
            String filePath = '/'+filename;
            FileSystem fs = FileSystem.get(new URI("hdfs://master:9000"), new Configuration());

            Path hdfsPath = new Path(filePath);
            if(fs.delete(hdfsPath,false)){
                System.out.println("File "+ filePath +" has been deleted successfully!");
            }
        }catch(Exception e) {
            e.printStackTrace();
        }
    }
    public static void DeleteDir(String dirname){
        try {
            String dirPath = '/'+dirname;
            FileSystem fs = FileSystem.get(new URI("hdfs://master:9000"), new Configuration());
            Path hdfsPath = new Path(dirPath);
            if(fs.delete(hdfsPath,true)){
                System.out.println("Directory "+ dirPath +" has been deleted successfully!");
            }
        }catch(Exception e) {
            e.printStackTrace();
        }
    }
    public static void ListFile(String dirnname){
        try {
            String filePath = dirnname;
            FileSystem fs = FileSystem.get(new URI("hdfs://master:9000"), new Configuration());
            Path srcPath = new Path(filePath);
            FileStatus[] stats = fs.listStatus(srcPath);
            Path[] paths = FileUtil.stat2Paths(stats);
            for(Path p : paths)
                System.out.println(p.getName());
        }catch(Exception e) {
            e.printStackTrace();
        }
    }
    public static void Rename(String sourcePath,String targetPath){
        try {
            String srcStrPath1 = sourcePath;
            String dstStrPath1= targetPath;
            //System.out.println(srcStrPath1);
            //System.out.println(dstStrPath1);
            FileSystem fs = FileSystem.get(new URI("hdfs://master:9000"), new Configuration());
            Path srcPath = new Path(srcStrPath1);
            Path dstPath = new Path(dstStrPath1);
            if(fs.rename(srcPath,dstPath)) {
                System.out.println("rename from " + srcStrPath1 + " to " + dstStrPath1 + "successfully!");
            }
        }catch(Exception e) {
            e.printStackTrace();
        }
    }
    public static void UploadFile(String sourcePath,String targetPath){
        ///headless/Desktop/workspace/hdfs_op/Y02014452.txt
        try {
            String srcStrPath = sourcePath;
            String dstStrPath = targetPath;
            FileSystem fs = FileSystem.get(new URI("hdfs://master:9000"), new Configuration());
            Path srcPath = new Path(srcStrPath);
            Path dstPath = new Path(dstStrPath);
            fs.copyFromLocalFile(srcPath,dstPath);
            System.out.println(srcStrPath+"upload finished");
        }catch(Exception e) {
            e.printStackTrace();
        }
    }
    public static void UploadDir(String sourcePath,String targetPath){
        try {
            String srcStrPath = sourcePath;
            String dstStrPath = targetPath;
            FileSystem fs = FileSystem.get(new URI("hdfs://master:9000"), new Configuration());
            Path srcPath = new Path(srcStrPath);
            Path dstPath = new Path(dstStrPath);
            fs.copyFromLocalFile(srcPath,dstPath);
            System.out.println(srcStrPath+"upload finished");
        }catch(Exception e) {
            e.printStackTrace();
        }
    }
    public static void Findfile(String targefile,String sourcePath){
        boolean flag=false;
        try {
            FileSystem fs = FileSystem.get(new URI("hdfs://master:9000"), new Configuration());
            Path srcPath = new Path(sourcePath);
            FileStatus[] stats = fs.listStatus(srcPath);
            for(FileStatus status:stats){
                if(status.isDirectory()){
                    String dirPath=status.getPath().toString();
                    String target=targefile;
                    Findfile(target,dirPath);
                }else {
                    if(status.getPath().getName().equals(targefile)) {
                        System.out.println("filename:" + status.getPath());
                        flag=true;
                    }
                }
            }
        }catch (Exception e){
            e.printStackTrace();
        }
    }
}