尚硅谷 Hadoop HDFS Java客户端环境准备及测试 遇到的坑

首先，贴一下要自己新建的文件 log4j.properties 的文件内容

log4j.rootLogger=INFO, stdout  
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d %p [%c] - %m%n
log4j.appender.logfile=org.apache.log4j.FileAppender  
log4j.appender.logfile.File=target/spring.log 
log4j.appender.logfile.layout=org.apache.log4j.PatternLayout
log4j.appender.logfile.layout.ConversionPattern=%d %p [%c] - %m%n

如果遇到这个错误提示：

2020-09-02 13:47:10,476 WARN [org.apache.hadoop.util.NativeCodeLoader] - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable

不用在意，这个提示不影响我们测试Hadoop HDFS的。也可以把他去掉，只需要在上面的log4j.properties文件中加上：

log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR

如果你遇到了这个错误提示：

Exception in thread "main" java.lang.UnsupportedOperationException

有可能是URI地址写错了：
不应该是：new URI("http://hadoop101:9000")
而应该是：new URI("hdfs://hadoop101:9000")
此处可参考文章 https://blog.csdn.net/weixin_34362790/article/details/91907387

最后，贴上我的测试代码：

package com.thomas.hdfs;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;
import org.junit.Test;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URISyntaxException;

public class HdfsClientDemo2 {

    public static void main(String[] args) throws IOException, URISyntaxException, InterruptedException {
        System.out.println("demo start :::::::");

        FileSystem fileSystem = getFileSystem();
        fileSystem.mkdirs(new Path("/com/lisa"));
//        fileSystem.copyToLocalFile(new Path("/user/thomas/input/hadoop-3.1.3.tar.gz"), new Path("/Users/thomas/Downloads/123"));
        fileSystem.close();
        System.out.println("end........");

    }

    @Test // 1文件上传
    public void testCopyFromLocalFile() throws InterruptedException, IOException, URISyntaxException {
        FileSystem fileSystem = getFileSystem();
        fileSystem.copyFromLocalFile(new Path("/Users/thomas/Downloads/az-thomas.ovpn"), new Path("/com/345.txt"));
        fileSystem.close();
    }

    @Test // 2文件下载
    public void testCopyToLocalFile() throws InterruptedException, IOException, URISyntaxException {
        FileSystem fileSystem = getFileSystem();
        fileSystem.copyToLocalFile(new Path("/com/123.txt"), new Path("/Users/thomas/Downloads/111"));
        fileSystem.close();
    }

    @Test // 3测试删除
    public void testDelete() throws InterruptedException, IOException, URISyntaxException {
        FileSystem fileSystem = getFileSystem();
        fileSystem.delete(new Path("/com/345.txt"), true);
        fileSystem.delete(new Path("/com/lisa"), true);
        fileSystem.close();
    }

    @Test // 4修改名称
    public void testRename() throws InterruptedException, IOException, URISyntaxException {
        FileSystem fileSystem = getFileSystem();
        fileSystem.rename(new Path("/com/123.txt"), new Path("/com/999.txt"));
        fileSystem.rename(new Path("/com/thomas"), new Path("/com/lisa"));
        fileSystem.close();
    }

    @Test // 4文件详情
    public void testFileStatus() throws InterruptedException, IOException, URISyntaxException {
        FileSystem fileSystem = getFileSystem();
//        FileStatus fileStatus = fileSystem.getFileStatus(new Path("/com/999.txt"));
        RemoteIterator<LocatedFileStatus> locatedFileStatusRemoteIterator = fileSystem.listFiles(new Path("/"), true);
        while (locatedFileStatusRemoteIterator.hasNext()){
            LocatedFileStatus fileStatus = locatedFileStatusRemoteIterator.next();

            System.out.println("文件名：" + fileStatus.getPath().getName());
            System.out.println("文件大小：" + fileStatus.getLen());
            System.out.println("文件权限：" + fileStatus.getPermission());
            System.out.println("文件组：" + fileStatus.getGroup());
            System.out.println("文件owner：" + fileStatus.getOwner());
            System.out.println("文件path：" + fileStatus.getPath());
            System.out.println("文件accessTime：" + fileStatus.getAccessTime());
            System.out.println("文件getBlockSize：" + fileStatus.getBlockSize());
            System.out.println("getModificationTime：" + fileStatus.getModificationTime());
            System.out.println("getReplication：" + fileStatus.getReplication());
            System.out.println("getPath().getParent()：" + fileStatus.getPath().getParent());

            BlockLocation[] blockLocations = fileStatus.getBlockLocations();
            for (BlockLocation blockLocation : blockLocations){
                String[] hosts = blockLocation.getHosts();
                for (String host : hosts){
                    System.out.println(host);
                }
            }
            System.out.println("-------文件分割线------");
        }
        fileSystem.close();
    }

    @Test // 4判断是文件还是目录
    public void testListFiles() throws InterruptedException, IOException, URISyntaxException {
        FileSystem fileSystem = getFileSystem();
        FileStatus[] fileStatuses = fileSystem.listStatus(new Path("/"));
        for (FileStatus fileStatus : fileStatuses) {
            if (fileStatus.isFile()){
                System.out.println(fileStatus.getPath().getName() + ":" + fileStatus.isFile());
            } else {
                System.out.println(fileStatus.getPath().getName() + ":" + fileStatus.isDirectory());
            }
        }
        fileSystem.close();
    }

    @Test // 5 IO操作文件上传
    public void testIOPush() throws InterruptedException, IOException, URISyntaxException {
        // 1获取对象
        FileSystem fileSystem = getFileSystem();
        // 2获取输入流
        FileInputStream fileInputStream = new FileInputStream(new File("/Users/thomas/Downloads/lisa.xlsx"));
        // 3获取输出流
        FSDataOutputStream fsDataOutputStream = fileSystem.create(new Path("/com/lisa4.xlsx"));
        // 4流的对考
//        Configuration configuration = new Configuration(); // 经测试，随便一个configuration都可以？！
//        IOUtils.copyBytes(fileInputStream, fsDataOutputStream, configuration);
        IOUtils.copyBytes(fileInputStream, fsDataOutputStream, fileSystem.getConf());
        // 5关闭资源 // 输入流输出流，经测试，不关闭也没事
        IOUtils.closeStream(fsDataOutputStream);
        IOUtils.closeStream(fileInputStream);
        fileSystem.close();
    }

    @Test // 6 IO操作文件下载
    public void testIOPull() throws InterruptedException, IOException, URISyntaxException {
        // 1获取对象
        FileSystem fileSystem = getFileSystem();
        // 2获取输入流
        FSDataInputStream open = fileSystem.open(new Path("/com/lisa4.xlsx"));
        // 3获取输出流
        FileOutputStream fileOutputStream = new FileOutputStream(new File("/Users/thomas/Downloads/lisa1.xlsx"));
        // 4流的对考
        IOUtils.copyBytes(open, fileOutputStream, fileSystem.getConf());
        // 5关闭资源 // 输入流输出流，经测试，不关闭也没事
        IOUtils.closeStream(fileOutputStream);
        IOUtils.closeStream(open);
        fileSystem.close();
    }

    @Test // 7 定位读取文件
    public void testReadFile() throws InterruptedException, IOException, URISyntaxException {
        // 1获取对象
        FileSystem fileSystem = getFileSystem();
        // 2获取输入流
        FSDataInputStream open = fileSystem.open(new Path("/user/thomas/input/hadoop-3.1.3.tar.gz"));
        // 3获取输出流
        FileOutputStream fileOutputStream = new FileOutputStream(new File("/Users/thomas/Downloads/hadoop.tar.gz.1"));
        // 4流的对考
        byte[] bytes = new byte[1024];
        for (int i=0; i<1024*128; i++){
            open.read(bytes);
            fileOutputStream.write(bytes);
        }
        // 5关闭资源 // 输入流输出流，经测试，不关闭也没事
        IOUtils.closeStream(fileOutputStream);
        IOUtils.closeStream(open);
        fileSystem.close();
    }

    @Test // 8 定位读取文件seek
    public void testReadSeek() throws InterruptedException, IOException, URISyntaxException {
        // 1获取对象
        FileSystem fileSystem = getFileSystem();
        // 2获取输入流
        FSDataInputStream open = fileSystem.open(new Path("/user/thomas/input/hadoop-3.1.3.tar.gz"));
        // 3获取输出流
        FileOutputStream fileOutputStream = new FileOutputStream(new File("/Users/thomas/Downloads/hadoop.tar.gz.2"));
        // 4流的对考
        open.seek(1024*1024*128);
        IOUtils.copyBytes(open, fileOutputStream, fileSystem.getConf());
        // 5关闭资源 // 输入流输出流，经测试，不关闭也没事
        IOUtils.closeStream(fileOutputStream);
        IOUtils.closeStream(open);
        fileSystem.close();
    }

    public static FileSystem getFileSystem() throws URISyntaxException, IOException, InterruptedException {
        Configuration configuration = new Configuration();
        configuration.set("fs.defaultFS", "hdfs://hadoop101:9000");
//        configuration.set("dfs.replication", "2"); // 参数优先级：代码指定 > hdfs-site.xml配置 > Hadoop默认配置
        FileSystem fileSystem = FileSystem.get(configuration);
        // 因为本机用户就是thomas，所以不用再指定thomas为用户了：
//        FileSystem fileSystem = FileSystem.get(new URI("hdfs://hadoop101:9000"), configuration, "thomas");
        return fileSystem;
    }
}

贴一下测试用到到hdfs-site.xml

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
    <property>
        <name>dfs.replication</name>
        <value>3</value>
    </property>

</configuration>