Hadoop 3.2.0 idea 开发环境搭建 与 HDFS读取写入API操作

首先需要搭建好hadoop服务器

请参考: Hadoop 3.2.0 完全分布式集群搭建 与 WordCount 运行

新建maven项目,添加hadoop相关jar包

        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>3.2.0</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-hdfs</artifactId>
            <version>3.2.0</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>3.2.0</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-mapreduce-client-core</artifactId>
            <version>3.2.0</version>
        </dependency>
        <dependency>
            <groupId>commons-logging</groupId>
            <artifactId>commons-logging</artifactId>
            <version>1.2</version>
        </dependency>

编写HDFS工具类

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

import java.net.URI;

/**
 * @author LionLi
 * @date 2019-2-7
 */
public class HDFSUtils {

    /** hdfs服务器地址 */
    private static final String hdfsUrl = "hdfs://192.168.0.200:8020/";
    /** 访问用户 */
    private static final String username = "root";

    public static void main(String[] args) throws Exception {
        String filePath = "/user/root/demo.txt";
        StringBuilder sb = new StringBuilder();
        for (int i = 1; i < 100001; i++) {
            sb.append("hadoop demo " + i + "\r\n");
        }
        HDFSCreateAndWriteFileValue(hdfsUrl,username, filePath,sb.toString());
        HDFSReadFileValue(hdfsUrl,username, filePath);
    }

    /**
     * 读取文件内容
     * @param url 服务器地址
     * @param user 访问用户
     * @param filePath 文件路径
     */
    private static void HDFSReadFileValue(
            String url,String user, String filePath) throws Exception {
        Configuration conf = new Configuration();
        try (FileSystem fileSystem = FileSystem.get(new URI(url), conf, user)) {
            Path demo = new Path(filePath);
            if (fileSystem.exists(demo)) {
                System.out.println("/user/root/demo.txt 文件存在");
                FSDataInputStream fsdis = fileSystem.open(demo);
                byte[] bytes = new byte[fsdis.available()];
                fsdis.readFully(bytes);
                System.out.println(new String(bytes));
            } else {
                System.out.println("/user/root/demo.txt 文件不存在");
            }
        }
    }

    /**
     * 创建并写入文件内容
     * @param url 服务器地址
     * @param user 访问用户
     * @param filePath 需要创建的文件路径
     * @param value 需要写入的值
     */
    private static void HDFSCreateAndWriteFileValue(
            String url,String user,String filePath,String value) throws Exception {
        Configuration conf = new Configuration();
        try (FileSystem fileSystem = FileSystem.get(new URI(url), conf, user)) {
            Path demo = new Path(filePath);
            if (fileSystem.exists(demo)) {
                System.out.println("/user/root/demo.txt 文件存在");
                if (fileSystem.delete(demo, false)) {
                    if (fileSystem.exists(demo)) {
                        System.out.println("/user/root/demo.txt 文件删除失败");
                    } else {
                        System.out.println("/user/root/demo.txt 文件删除成功");
                        FSDataOutputStream fsdos = fileSystem.create(demo);
                        fsdos.write(value.getBytes());
                    }
                }
            } else {
                System.out.println("/user/root/demo.txt 文件不存在");
                FSDataOutputStream fsdos = fileSystem.create(demo);
                fsdos.write(value.getBytes());
            }
        }
    }
}

测试运行

测试成功