hadoop之旅5-idea通过maven搭建hdfs环境

相信大家通过之前的做法都已经搭建起了一个hadoop的开发环境。今天带大家通过java api来访问hdfs文件系统

首先启动hadoop集群

start-dfs.sh
或者
start-all.sh  //一键启动hadoop集群和yarn集群
复制代码

打开idea

在pom.xml文件里加入hadoop的依赖，我这里使用的是我搭建的一样版本的依赖 hadoop 2.7.3

<properties>
	<hadoop.version>2.7.3</hadoop.version>
</properties>

<dependency>
	<groupId>org.apache.hadoop</groupId>
	<artifactId>hadoop-common</artifactId>
	<version>${hadoop.version}</version>
</dependency>
<dependency>
	<groupId>org.apache.hadoop</groupId>
	<artifactId>hadoop-client</artifactId>	
	<version>${hadoop.version}</version>
</dependency>
<dependency>
        <groupId>org.apache.hadoop</groupId>
	<artifactId>hadoop-hdfs</artifactId>
	<version>${hadoop.version}</version>
</dependency>
复制代码

Java api在hdfs上创建一个文件目录

//创建配置文件
Configuration conf = new Configuration();
 //windows下无法找到对应的环境变量，需要设置。把hadoop解压下来的根目录
System.setProperty("hadoop.home.dir", "F:\\linux\\hadoop-2.7.3");
// 指定hadoop fs的地址
conf.set("fs.defaultFS", "hdfs://master:9000");
//定义访问的根目录
String userRootPath = "/userSpace"
//拿到文件操作对象
FileSystem fs = FileSystem.get(conf);
//创建一个path对象，hdfs上的目录都需要用path对象来访问
Path dir = new Path(userRootPath);  //表示根目录下的 userSpace目录
boolean result = fs.mkdirs(dir);
if(result){
    System.out.println("创建目录成功！");
}
复制代码

其实就和java访问文件一样的操作类似，非常简单，不过操作hdfs主要是通过FileSystem类。下面给大家贴一下完整代码

package com.mmcc.springboothadoop.hadoop;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;

import java.io.ByteArrayOutputStream;
import java.io.IOException;

public class HdfsUtils {
    private static Configuration conf = new Configuration();
    public static String userRootPath = "/userSpace";

    static {
        //windows下无法找到对应的环境变量，需要设置
        System.setProperty("hadoop.home.dir", "F:\\linux\\hadoop-2.7.3");
        // 指定hadoop fs的地址
        conf.set("fs.defaultFS", "hdfs://master:9000");
    }

    //判断路径是否存在
    public static boolean exists(String path) throws IOException {
        FileSystem fileSystem = FileSystem.get(conf);
        return fileSystem.exists(new Path(path));
    }

    //创建文件
    public static void createFile(String filePath, byte[] contents) throws IOException {
        FileSystem fileSystem = FileSystem.get(conf);
        Path path = new Path(filePath);
        FSDataOutputStream fdo = fileSystem.create(path);
        fdo.write(contents);
        fdo.close();
        fileSystem.close();

    }

    /**
     * 创建文件
     *
     * @param filePath
     * @param fileContent
     * @throws IOException
     */
    public static void createFile(String filePath, String fileContent)
            throws IOException {
        createFile(filePath, fileContent.getBytes());
    }

    //从本地复制到hdfs上
    public static void copyFromLocalFile(String localFilePath, String remoteFilePath) throws IOException {
        FileSystem fs = FileSystem.get(conf);
        Path localPath = new Path(localFilePath);
        Path remotePath = new Path(remoteFilePath);
        fs.copyFromLocalFile(false, true, localPath, remotePath);
        fs.close();
    }


    /**
     * 删除目录或文件
     *
     * @param remoteFilePath
     * @param recursive
     * @return
     * @throws IOException
     */
    public static boolean deleteFile(String remoteFilePath, boolean recursive)
            throws IOException {
        FileSystem fs = FileSystem.get(conf);
        boolean result = fs.delete(new Path(remoteFilePath), recursive);
        fs.close();
        return result;
    }
    /**
     * 删除目录或文件(如果有子目录,则级联删除)
     *
     * @param remoteFilePath
     * @return
     * @throws IOException
     */
    public static boolean deleteFile(String remoteFilePath) throws IOException {
        return deleteFile(remoteFilePath, true);
    }
    /**
     * 文件重命名
     *
     * @param oldFileName
     * @param newFileName
     * @return
     * @throws IOException
     */
    public static boolean renameFile(String oldFileName, String newFileName)
            throws IOException {
        FileSystem fs = FileSystem.get(conf);
        Path oldPath = new Path(oldFileName);
        Path newPath = new Path(newFileName);
        boolean result = fs.rename(oldPath, newPath);
        fs.close();
        return result;
    }
    /**
     * 创建目录
     *
     * @param dirName
     * @return
     * @throws IOException
     */
    public static boolean createDirectory(String dirName) throws IOException {
        FileSystem fs = FileSystem.get(conf);
        Path dir = new Path(dirName);
        boolean result = false;
        if (!fs.exists(dir)) {
            result = fs.mkdirs(dir);
        }
        fs.close();
        return result;
    }

    //列出指定路径下的文件
    public static  RemoteIterator<LocatedFileStatus> listFiles(String dirPath,boolean recursive) throws IOException {
        FileSystem fs = FileSystem.get(conf);
        RemoteIterator<LocatedFileStatus> listFiles = fs.listFiles(new Path(dirPath), recursive);//不进行递归
        fs.close();
        return listFiles;
    }
    /**
     * 列出指定路径下的文件（非递归）
     *
     * @param basePath
     * @return
     * @throws IOException
     */
    public static RemoteIterator<LocatedFileStatus> listFiles(String basePath)
            throws IOException {
        FileSystem fs = FileSystem.get(conf);
        RemoteIterator<LocatedFileStatus> remoteIterator = fs.listFiles(
                new Path(basePath), false);
        fs.close();
        return remoteIterator;
    }

    /**
     * 列出指定目录下的文件\子目录信息（非递归）
     *
     * @param dirPath
     * @return
     * @throws IOException
     */
    public static FileStatus[] listStatus(String dirPath) throws IOException {
        FileSystem fs = FileSystem.get(conf);
        FileStatus[] fileStatuses = fs.listStatus(new Path(dirPath));
        fs.close();
        return fileStatuses;
    }

    //读取文件内容
    public static byte[] readFile(String filePath) throws IOException {
        byte[] fileContent = null;
        FileSystem fs = FileSystem.get(conf);
        Path path = new Path(filePath);

        if (fs.exists(path)){
            FSDataInputStream fsin = fs.open(path);
            ByteArrayOutputStream bos = new ByteArrayOutputStream();
            IOUtils.copyBytes(fsin,bos,conf);
            fileContent = bos.toByteArray();
        }
        return fileContent;
    }

    //下载hdfs上的文件
    public static void download(String remote,String local) throws IOException {
        FileSystem fs = FileSystem.get(conf);
        //远程hdfs上的文件
        Path remotePath = new Path(remote);
        //本地的文件
        Path localPath = new Path(local);
        fs.copyToLocalFile(remotePath,localPath);
        fs.close();
    }
}
复制代码