Java操作Hadoop的HDFS文件系统

java操作Hadoop的HDFS文件系统

本教程适用于已搭建Hadoop集群的环境下

项目结构

1.构建项目，导入maven相关依赖

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>atguigu</groupId>
    <artifactId>HDFSClient</artifactId>
    <version>1.0-SNAPSHOT</version>
    <dependencies>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>3.1.3</version>
        </dependency>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.12</version>
        </dependency>
        <dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-log4j12</artifactId>
            <version>1.7.30</version>
        </dependency>
    </dependencies>


</project>

2.log4j.properties配置日志打印

log4j.rootLogger=INFO, stdout  
log4j.appender.stdout=org.apache.log4j.ConsoleAppender  
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout  
log4j.appender.stdout.layout.ConversionPattern=%d %p [%c] - %m%n  
log4j.appender.logfile=org.apache.log4j.FileAppender  
log4j.appender.logfile.File=target/spring.log  
log4j.appender.logfile.layout=org.apache.log4j.PatternLayout  
log4j.appender.logfile.layout.ConversionPattern=%d %p [%c] - %m%n

3.hdfs-site.xml用于测试配置的优先级

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<configuration>
    <property>
        <name>dfs.replication</name>
        <value>1</value>
    </property>
</configuration>

4.编写Java测试类

package com.atguigu.hdfs;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Arrays;

/**
 * @author TANGSHUAI
 * @version 1.0
 * @date 2022-08-21 17:40
 * 1.获取一个客户端对象
 * 2.执行相关命令操作
 * 3.关闭资源
 */
public class HDFSClient {
    FileSystem fileSystem;

    @Before
    public void init() throws IOException, InterruptedException, URISyntaxException {
        //连接集群nn地址
        URI uri = new URI("hdfs://hadoop102:8020");
        //创建一个配置文件
        Configuration configuration=new Configuration();
        String user="root";
        //获取客户端对象
        fileSystem= FileSystem.get(uri, configuration,user);
    }

    @After
    public void close() throws IOException {
        //关闭资源
        fileSystem.close();
    }

    @Test
    public void testMkdir() throws URISyntaxException, IOException, InterruptedException {
        //创建一个文件夹
        fileSystem.mkdirs(new Path("/xiyou/huaguoshan") );
    }

    /**
     * 参数优先级由低到高：hdfs-default.xml < hdfs-site.xml < 项目resources < 代码内的配置
     * @throws IOException
     */
    @Test
    public void testPut() throws IOException {
        //文件上传，参数一：表示删除原数据，参数二：是否允许覆盖，参数三：原数据路径，参数四:目的地路径
        fileSystem.copyFromLocalFile(false, false, new Path("D:\\zhubajie.txt"),new Path("/xiyou/huaguoshan"));
    }

    @Test
    public void testGet() throws IOException {
        //文件下载,参数一：原文件是否删除，参数二：原文件路径HDFS，参数三：目标地址路径windows，参数四：是否加效验
        fileSystem.copyToLocalFile(false, new Path("/xiyou/huaguoshan"),new Path("D:\\tangseng") ,false );
    }

    @Test
    public void testRm() throws IOException {
        //文件删除,参数一：要删除的文件路径，参数二：是否递归删除
        fileSystem.delete(new Path("/xiyou/huaguoshan/sunwukong.txt"), true);
    }

    @Test
    public void testMv() throws IOException {
        //文件更名和移动,参数一：原文件路径，参数二：目标文件路径
        fileSystem.rename(new Path("/xiyou/huaguoshan/zhubajie.txt"), new Path("/xiyou/huaguoshan/zhubajie2.txt"));
    }

    @Test
    public void fileDetail() throws IOException {
        //获取所有文件信息
        RemoteIterator<LocatedFileStatus> list = fileSystem.listFiles(new Path("/"), true);
        while (list.hasNext()){
            LocatedFileStatus fileStatus = list.next();
            System.out.println(fileStatus.getPath());
            // 获取块信息
            BlockLocation[] blockLocations = fileStatus.getBlockLocations();
            System.out.println(Arrays.toString(blockLocations));

        }

    }

    @Test
    public void testListStatus() throws IOException, InterruptedException, URISyntaxException{
        // 判断是文件还是目录
        FileStatus[] listStatus = fileSystem.listStatus(new Path("/"));
        for (FileStatus fileStatus : listStatus) {
            // 如果是文件
            if (fileStatus.isFile()) {
                System.out.println("文件:"+fileStatus.getPath().getName());
            }else {
                System.out.println("目录:"+fileStatus.getPath().getName());
            }
        }
    }

}

根据以上操作查看HDFS文件系统的变化