hadoop3的Java接口工具类

最新推荐文章于 2022-10-05 17:17:00 发布

froeverything

最新推荐文章于 2022-10-05 17:17:00 发布

阅读量577

点赞数 1

文章标签： java hadoop hdfs

本文链接：https://blog.csdn.net/m0_49926323/article/details/126671674

版权

hadoop3的Java接口工具类

准备工作

配置好对应版本hadoop winutils

pom文件

     <dependency>
                <groupId>org.apache.hadoop</groupId>
                <artifactId>hadoop-common</artifactId>
                <version>${hadoop.version}</version>
            </dependency>
            <dependency>
                <groupId>org.apache.hadoop</groupId>
                <artifactId>hadoop-hdfs</artifactId>
                <version>${hadoop.version}</version>

            </dependency>
            <dependency>
                <groupId>org.apache.hadoop</groupId>
                <artifactId>hadoop-client</artifactId>
                <version>${hadoop.version}</version>
            </dependency>
            <dependency>
                <groupId>org.apache.hadoop</groupId>
                <artifactId>hadoop-mapreduce-client-core</artifactId>
                <version>${hadoop.version}</version>
            </dependency>
      <dependency>
                <groupId>org.apache.logging.log4j</groupId>
                <artifactId>log4j-core</artifactId>
                <version>2.18.0</version>
            </dependency>

java工具类


/**
 * @author wangqinglu
 * &#064;date  2022/9/2
 */
public class HDFSUtils {

    /** hdfs服务器地址 */
    private static final String hdfsUrl = "hdfs://192.168.228.132:8020/";
    /** 访问用户 */
    private static final String username = "root";
    /** 日志接口 */
   private static final  Configuration conf = new Configuration();



   /**
     * 建文件夹
     * @param hdfsPath 文件路径
     */
    public  static void makdirFolder(String hdfsPath) throws  Exception {
        try (FileSystem hdfs =  FileSystem.get(new URI(hdfsUrl),conf,username)){
            Path path = new Path(hdfsPath);
            if (hdfs.exists(path)){
                System.out.println("文件夹已存在");
            }else {hdfs.mkdirs(path);
                System.out.println("创建文件成功,路径"+hdfsPath);}
        }}

    /**
     * 上传文件
     * @param hdfsPath 文件路径
     * @param localPath 本地文件路径
     */
    public static void putFile(String localPath,String hdfsPath)throws  Exception {
        try (FileSystem hdfs =  FileSystem.get(new URI(hdfsUrl),conf,username)){
            hdfs.copyFromLocalFile(false,true,new Path(localPath),new Path(hdfsPath));
            System.out.println("上传文件成功，路径"+hdfsPath);
        }
    }

    /**
     * 下载文件
     * @param hdfsPath 文件路径
     * @param localPath 本地文件路径
     */
    public static void getFile(String hdfsPath,String localPath)throws  Exception {
        try (FileSystem hdfs =  FileSystem.get(new URI(hdfsUrl),conf,username)){
            hdfs.copyToLocalFile(false,new Path(hdfsPath),new Path(localPath),true);
            System.out.println("文件下载成功");
        }
    }

    /**
     * 文件删除
     * @param hdfsPath 文件路径
     */
    public static void deleteFile(String hdfsPath)throws  Exception {
        try (FileSystem hdfs =  FileSystem.get(new URI(hdfsUrl),conf,username)){
            hdfs.delete(new Path(hdfsPath),true);
            System.out.println("文件删除成功");
        }}

    /**
     * 移动文件，可修改名称
     *@param newHdfsPile 文件新路径
     * @param hdfsPath 文件路径
     */
    public static void mvFile(String hdfsPath,String newHdfsPile)throws  Exception {
        try (FileSystem hdfs =  FileSystem.get(new URI(hdfsUrl),conf,username)){
            hdfs.rename(new Path(hdfsPath),new Path(newHdfsPile));
            System.out.println("文件移动成功,新路径"+newHdfsPile);
        }}

    /**
     * 获取文件详细信息
     *
     * @param hdfsPath 文件路径
     */
    public static void fileDetail(String hdfsPath)throws  Exception {
       try (FileSystem hdfs =  FileSystem.get(new URI(hdfsUrl),conf,username)){
           RemoteIterator<LocatedFileStatus> listFiles = hdfs.listFiles(new Path(hdfsPath), true);
           while (listFiles.hasNext()) {
               LocatedFileStatus fileStatus = listFiles.next();
               System.out.println("========"+fileStatus.getPath()+"========");
               System.out.println(fileStatus.getPermission());
               System.out.println(fileStatus.getOwner());
               System.out.println(fileStatus.getGroup());
               System.out.println(fileStatus.getLen());
               System.out.println(fileStatus.getModificationTime());
               System.out.println(fileStatus.getReplication());
               System.out.println(fileStatus.getBlockSize());
               System.out.println(fileStatus.getPath().getName());
               BlockLocation[] blockLocations = fileStatus.getBlockLocations();
               System.out.println(Arrays.toString(blockLocations));
           }}}
    /**
     * 判断是否是文件夹
     *
     * @param hdfsPath 文件路径
     */
    public static void judgeFile(String hdfsPath)throws  Exception {
        try (FileSystem hdfs =  FileSystem.get(new URI(hdfsUrl),conf,username)){
            FileStatus[] listStatus = hdfs.listStatus(new Path(hdfsPath));
            for (FileStatus status:listStatus){
                if (status.isFile()) System.out.println("文件: " + status.getPath().getName());
                 else System.out.println("目录: "+ status.getPath().getName());
            }
        }}

    /**
     * 读取文件内容
     *
     * @param filePath 文件路径
     */
    public static void HDFSReadFileValue(String filePath) throws Exception {
        try (FileSystem fileSystem = FileSystem.get(new URI(hdfsUrl), conf, username)) {
            Path demo = new Path(filePath);
            if (fileSystem.exists(demo)) {
                System.out.println("文件存在");
                FSDataInputStream fsdis = fileSystem.open(demo);
                byte[] bytes = new byte[fsdis.available()];
                fsdis.readFully(bytes);
                System.out.println(new String(bytes));
            } else {
                System.out.println(" 文件不存在");
            }
        }
    }