HdfsAPI基础

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.junit.Test;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

public class HdfsAPI {

    @Test
    public void initHdfs() throws IOException {
        //1.实例化配置文件
        Configuration conf = new Configuration();

        //2.获取文件系统
        FileSystem fs = FileSystem.get(conf);

        //3.打印文件系统
        System.out.println(fs.toString());

    }

    @Test
    /**
     * 上传
     */
    public void putFileHdfs() throws URISyntaxException, IOException, InterruptedException {
        //1.实例化配置文件
        //alt+enter 导包
        Configuration conf = new Configuration();
        //设置参数
        conf.set("dfs.replication","2");
        //2.获取文件系统
        FileSystem fs = FileSystem.get(new URI("hdfs://bigdata111:9000"), conf, "root");

        //3.定义文件输入路径
        Path src = new Path("F:\\student.txt");

        //4.定义文件输出路径，上传hdfs
        Path out = new Path("hdfs://bigdata111:9000/");

        //5.上传
        fs.copyFromLocalFile(src,out);
        //6.关闭文件系统
        fs.close();

        System.out.println("文件上传成功");
    }


    /**
     * 下载
     * @throws URISyntaxException
     * @throws IOException
     * @throws InterruptedException
     */
    @Test
    public void getFileHdfs() throws URISyntaxException, IOException, InterruptedException {
        //实例化配置
        Configuration conf = new Configuration();
        //获取文件系统
        FileSystem fs = FileSystem.get(new URI("hdfs://bigdata111:9000"),conf,"root");

        //定义下载地址，hdfs
        Path src = new Path("/test");
        //定义下载到win的地址
        Path out = new Path("F:\\test1.txt");

        //下载
        //delSrc 是否删除源文件
        //src hdfs路径
        //dst win路径
        //useRawLocalFileSystem 是否进行校验
        fs.copyToLocalFile(true,src,out,true);

        //关闭
        fs.close();

        System.out.println("下载成功！");

    }

    /**
     * 创建目录
     * @throws URISyntaxException
     * @throws IOException
     * @throws InterruptedException
     */
    @Test
    public void mkdirHdfs() throws URISyntaxException, IOException, InterruptedException {
        //实例化配置
        Configuration conf = new Configuration();
        //获取文件系统
        FileSystem fs = FileSystem.get(new URI("hdfs://bigdata111:9000"), conf, "root");

        //hdfs里的路径
        Path path = new Path("/0805");

        //创建目录
        fs.mkdirs(path);

        fs.close();

        System.out.println("创建成功！");

    }

    @Test
    /**
     * 删除 hdfs文件
     */
    public void rmHdfs() throws URISyntaxException, IOException, InterruptedException {
        //实例化配置
        Configuration conf = new Configuration();
        //获取文件系统
        FileSystem fs = FileSystem.get(new URI("hdfs://bigdata111:9000"), conf, "root");

        //删除hdfs的路径
        Path path = new Path("/0805");
        //删除 (删除路径，是否递归)
        fs.delete(path, false);

        //关闭
        fs.close();

        System.out.println("删除成功!");
    }

    /**
     * 重命名
     */
    @Test
    public void reNameHdfs() throws URISyntaxException, IOException, InterruptedException {
        //实例化配置
        Configuration conf = new Configuration();
        //获取文件系统
        FileSystem fs = FileSystem.get(new URI("hdfs://bigdata111:9000"), conf, "root");

        //重命名
        fs.rename(new Path("/student.txt"), new Path("/person"));

        fs.close();

        System.out.println("修改名完成！");
    }

    /**
     * 查看文件详情
     */
    @Test
    public void readFileHdfs() throws URISyntaxException, IOException, InterruptedException {
        //实例化配置
        Configuration conf = new Configuration();
        //获取文件系统
        FileSystem fs = FileSystem.get(new URI("hdfs://bigdata111:9000"), conf, "root");

        //获取到目录里的所有文件的一个迭代器
        RemoteIterator<LocatedFileStatus> listfiles = fs.listFiles(new Path("/"), true);

        while (listfiles.hasNext()){
            //拿到每个文件的对象
            LocatedFileStatus file = listfiles.next();

            System.out.println("文件名字："+file.getPath().getName());
            System.out.println("块的大小："+file.getBlockSize());
            System.out.println("文件的大小："+file.getLen());

            BlockLocation[] blockLocations = file.getBlockLocations();
            for (BlockLocation block : blockLocations) {
                System.out.println("偏移量："+block.getOffset());
                String[] hosts = block.getHosts();
                for (String host : hosts) {
                    System.out.println(host);
                }
            }

            System.out.println("====================");
        }

        fs.close();
    }

    /**
     * 判断是否是文件1
     */
    @Test
    public void isFileHdfs() throws URISyntaxException, IOException, InterruptedException {
        //实例化配置
        Configuration conf = new Configuration();
        //获取文件系统
        FileSystem fs = FileSystem.get(new URI("hdfs://bigdata111:9000"), conf, "root");

        FileStatus[] fileStatuses = fs.listStatus(new Path("/"));
        for (FileStatus status : fileStatuses) {
            if (status.isFile()){
                System.out.println(status.getPath().getName()+"是文件");
            }else {
                System.out.println(status.getPath().getName()+"是文件夹");
            }
        }

        fs.close();
    }
}