HDFS调用API

·maven（版本管理）项目

<dependencies>
    <!--单元测试-->
    <dependency>
        <groupId>junit</groupId>
        <artifactId>junit</artifactId>
        <version>4.12</version>
    </dependency>
    <!--打印日志-->
    <dependency>
        <groupId>org.apache.logging.log4j</groupId>
        <artifactId>log4j-slf4j-impl</artifactId>
        <version>2.12.0</version>
    </dependency>
    <!--hadoop客户端-->
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-client</artifactId>
        <version>3.1.3</version>
    </dependency>
</dependencies>

public class hdfsClient {
    /**
     * 另外一台机器调用集群，对集群进行操作（客户端代码操作集群）
     *1.获取Hadoop客户端
     *2.进行操作命令（创建一个文件夹）
     *3.关闭资源
     */
    public void testmkdir(){
        
    }
}

public class hdfsClient {
    /**
     * 另外一台机器调用集群，对集群进行操作（客户端代码操作集群）
     *1.获取Hadoop客户端
     *2.进行操作命令（创建一个文件夹）
     *3.关闭资源
     */
 @Test
    public void testmkdir1() throws URISyntaxException, IOException, InterruptedException {
        //连接集群的namenode地址
        URI uri = new URI("hdfs://172.18.0.2:9000");
        //创建一个配置文件
        Configuration configuration = new Configuration();
        //登录用户
        String user = "root";
        //获取到客户对象
        FileSystem fs = FileSystem.get(uri,configuration,user);
 
        //创建一个文件夹
        fs.mkdirs(new Path("/youxiuderen"));
 
        //关闭资源
        fs.close();
    }

上传文件

    @Test
    public void testPut() throws URISyntaxException, IOException, InterruptedException {
        //连接集群的namenode的地址
        URI uri = new URI("hdfs://172.18.0.2:9000");
        //创建一个配置文件
        Configuration configuration = new Configuration();
        //登录用户
        String user =  "root";
        //获取客户端对象
        FileSystem fs = FileSystem.get(uri,configuration,user);
 
        //参数解读：参数1：表示删除源数据；参数 2：是否允许覆盖；参数3：源数据的路径；参数4：目的地路径（存放到hdfs系统的具体路径）
        fs.copyFromLocalFile(false,false,new Path("/root/IdeaProjects/hdfsClient/target/tiansong.txt"),new Path("/youxiuderen1"));
        //关闭资源
        fs.close();
    }

下载文件

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
 
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
 
public class hdfsClient {
    /**
     *
     */
    private FileSystem fs;
    @Before
    public void init() throws URISyntaxException, IOException, InterruptedException {
        URI uri = new URI("hdfs://172.18.0.2:9000");
        Configuration configuration = new Configuration();
        String user = "root";
        fs = FileSystem.get(uri,configuration,user);
    }
    @After
    public void  close() throws IOException {
        fs.close();
    }
    @Test
    public void testmkdir() throws URISyntaxException, IOException, InterruptedException {
        fs.mkdirs(new Path("/youxiuderen3"));
 
    }
    @Test
    public void testmkdir1() throws URISyntaxException, IOException, InterruptedException {
        fs.mkdirs(new Path("/youxiuderen4"));
 
    }
 
    @Test
    public void testPut() throws URISyntaxException, IOException, InterruptedException {
 
        fs.copyFromLocalFile(false, true, new Path("/root/IdeaProjects/hdfsClient/target/tiansong.txt"), new Path("/youxiuderen1"));
    }
 
    @Test
    public void testPut2() throws URISyntaxException, IOException, InterruptedException {
 
        fs.copyFromLocalFile(true, true, new Path("/root/IdeaProjects/hdfsClient/target/tiansong.txt"), new Path("/youxiuderen1"));
    }
    @Test
    public void testGet() throws IOException {
        //参数解读：小字 noolean delsrc：是否将源文件删除 ；Path src：要下载的文件路径 ；Path dst：将文件下载到本地服务器端的路径 ；boolean useRawLocalFileSystem是否开启校验
        fs.copyToLocalFile(false,new Path("/youxiuderen1/tiansong.txt"),new Path("/root/IdeaProjects/hdfsClient")，false);
    }
}

第一个false 第二个true 为覆盖

    @Test
    public void testPut() throws URISyntaxException, IOException, InterruptedException {
 
        fs.copyFromLocalFile(false, true, new Path("/root/IdeaProjects/hdfsClient/target/tiansong.txt"), new Path("/youxiuderen1"));
    }

    @Test
    public void testPut2() throws URISyntaxException, IOException, InterruptedException {
 
        fs.copyFromLocalFile(true, true, new Path("/root/IdeaProjects/hdfsClient/target/tiansong.txt"), new Path("/youxiuderen1"));
    }
}

第一个true 第二个true 删除本地文件 但是hdfs里面还有

    @Test
    public void testGet() throws IOException {
        fs.copyToLocalFile(false,new Path("/youxiuderen1/tiansong.txt"),new Path("/root/IdeaProjects/hdfsClient/target"));
    }

重新下载

crc文件 循环冗余

末尾true 校验 自动校验 没有crc后缀 

    @Test
    public void testGet() throws IOException {
        fs.copyToLocalFile(false,new Path("/youxiuderen1/tiansong1.txt"),new Path("/root/IdeaProjects/hdfsClient"),true);
    }

末尾为false 生成crc文件 

    @Test
    public void testGet() throws IOException {
        fs.copyToLocalFile(false,new Path("/youxiuderen1/tiansong.txt"),new Path("/root/IdeaProjects/hdfsClient"),false);
    }

Last Modified改变了    本地删除 hdfs里面有