hadoop-HDFS操作-shell端-java端需要的依赖-简单案例-修改idea格式化代码快捷键

1 shell端

就是在linux虚拟机上操作HDFS
常用命令如下:

hdfs -help--->查看帮助
不记得的查看帮助就可以了

2 java端

在IDEA中编写代码操作HDFS
依赖

<dependencies>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>3.2.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-hdfs</artifactId>
            <version>3.2.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-mapreduce-client-core</artifactId>
            <version>3.2.1</version>
        </dependency>

        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-mapreduce-client-common</artifactId>
            <version>3.2.1</version>
        </dependency>

        <!-- https://mvnrepository.com/artifact/com.alibaba/fastjson -->
        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>fastjson</artifactId>
            <version>1.2.68</version>
        </dependency>

        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
            <version>3.2.1</version>
        </dependency>
    </dependencies>

2.1 上传数据

/**
         * 1. 获取配置对象
         * 2. 获取文件系统对象
         * 3. 使用对象调用指定方法
         * 4. 关闭文件系统对象
         */
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.newInstance(new URI("hdfs://linux01:8020"), conf, "root");
        //上传多个路径下的文件
        fs.copyFromLocalFile(false,false,new Path[]{new Path("D://user.txt"),new Path("J://word.txt")},new Path("/hdfsjava"));
        //上传单个文件
        //fs.copyFromLocalFile(new Path(""),new Path(""));
        fs.close();

2.2 下载数据

		Configuration conf = new Configuration();
        FileSystem fs = FileSystem.newInstance(new URI("hdfs://linux01:8020"), conf, "root");
        //下载文件/文件夹
        fs.copyToLocalFile(new Path("/hdfsjava"),new Path("H://"));
        fs.close();

2.3 mkdir

Configuration conf = new Configuration();
        FileSystem fs = FileSystem.newInstance(new URI("hdfs://linux01:8020"), conf, "root");
        //也可以在创建文件夹的时候添加权限
        fs.mkdirs(new Path("/newpath"));
        fs.close();

2.4 rename

Configuration conf = new Configuration();
        FileSystem fs = FileSystem.newInstance(new URI("hdfs://linux01:8020"), conf, "root");
        fs.rename(new Path("/newpath"),new Path("/oldpath"));
        fs.close();

2.5 listFiles和listStatus

public static void main(String[] args) throws Exception {
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.newInstance(new URI("hdfs://linux01:8020"), conf, "root");
        /**
         * 参数一      路径
         * 参数二      是否递归获取--->递归就和linux命令中的ll一样 , 可以查看目录下的所有内容
         * 返回一个迭代器--->迭代器中的是目录下的文件和文件夹
         */
        RemoteIterator<LocatedFileStatus> iterator = fs.listFiles(new Path("/hdfsjava"), false);
        //查看目录下所有文件和文件夹的状态--->也可以通过只获取部分文件/文件夹的部分属性
        FileStatus[] fileStatuses = fs.listStatus(new Path("/"));
        for (FileStatus fileStatus : fileStatuses) {
            System.out.println(fileStatus);
        }
        System.out.println("-------------------------------");
        while (iterator.hasNext()){
            LocatedFileStatus file = iterator.next();
            if(file.isFile()){
                //如果是文件---获取文件所在路径
                Path filePath = file.getPath();
                //获取文件名
                String fileName = filePath.getName();
                //查看数据块的大小
                long blockSize = file.getBlockSize();
                //查看文件的权限
                FsPermission permission = file.getPermission();
                //获取数据所在的块
                BlockLocation[] blockLocations = file.getBlockLocations();
                for (BlockLocation blockLocation : blockLocations) {
                    //偏移量
                    long offset = blockLocation.getOffset();
                    //长度
                    long length = blockLocation.getLength();
                    //获取数据块存储的所有主机名
                    String[] hosts = blockLocation.getHosts();
                    //打印文件名
                    System.out.println(fileName);
                    for (String host : hosts) {
                        System.out.println(host);
                    }
                    System.out.println("------------------");
                }
            }
        }
    }

2.6 读写数据

读

		Configuration conf = new Configuration();
        FileSystem fs = FileSystem.newInstance(new URI("hdfs://linux01:8020"), conf, "root");
        //读取数据
        FSDataInputStream open = fs.open(new Path(""));
        //第一行数据 a b c
        //跳过a
        //open.seek(1);
        //跳过一个,读取第二个
        //int i = open.read();
        //包在缓冲字符流中逐行读取
        BufferedReader reader = new BufferedReader(new InputStreamReader(open));
        String line = null;
        while ((line = reader.readLine()) != null){
            System.out.println(line);
        }
        open.close();
        reader.close();
        fs.close();

写

Configuration conf = new Configuration();
        FileSystem fs = FileSystem.newInstance(new URI("hdfs://linux01:8020"), conf, "root");
        //hdfs 不支持并发写---随机写---随机修改
        //在文件末尾追加内容
        FSDataOutputStream append = fs.append(new Path("/hdfsjava/word.txt"));
        append.writeUTF("hello");
        //覆盖文件内容
        FSDataOutputStream fos = fs.create(new Path("/hdfsjava/user.txt"), true);
        fos.writeUTF("world");
        fos.close();
        append.close();
        fs.close();

修改idea格式化代码快捷键

File—>Settings—>搜索reformat—>ReFormat Code—>右键Add Keyboard Shortcut