JAVA实现HDFS文件夹目录下文件的解压缩

最新推荐文章于 2022-11-28 10:41:33 发布

斯特凡今天也很帅

最新推荐文章于 2022-11-28 10:41:33 发布

阅读量1.2k

点赞数 1

本文链接：https://blog.csdn.net/weixin_41311528/article/details/126371718

版权

Java 同时被 2 个专栏收录

81 篇文章 0 订阅

订阅专栏

Hadoop

16 篇文章 0 订阅

订阅专栏

一、核心代码

  public static void main(String[] args) {
        String inputPath= args[0];
        try {
		//1.新建一个Configuration对象
			Configuration configuration = new Configuration();
       		configuration.set("fs.defaultFS","hdfs://192.168.110.101:8020");
        //2.获取FileSystem对象
        	FileSystem fs = FileSystem.get(configuration);
            fs = FileSystem.get(conf);
       //3.使用gzip压缩方式    
            Class<GzipCodec> gzipCodecClass = GzipCodec.class;
            codec = ReflectionUtils.newInstance(gzipCodecClass, conf);
            switch (className) {
                case "Directory":
                    Directory(inputPath);
                    break;
                case "UNDirectory":
                    UNDirectory(inputPath);
                    break;
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
    private static void Directory(String inputPath) {
        try {
            FileStatus[] status = fs.listStatus(new Path(inputPath));
            for (FileStatus file : status) {
                System.out.println("====file"+file);
                System.out.println("====file.getPath()"+file.getPath());
                //得到输入流
                //输出文件路径
                String outputPath = file.getPath().toString().substring(0,  file.getPath().toString().length() - 4) + ".gz";
                System.out.println("====outputPath"+outputPath);
                //得到压缩输出流
                CompressionOutputStream outIO = codec.createOutputStream(fs.create(new Path(outputPath)));
                IOUtils.copyBytes(fs.open(file.getPath()), outIO, conf, true);
                System.out.println("COMPRESS SUCCESS:::" + outputPath);
                if (fs.exists(new Path(outputPath))) {
                    fs.delete(new Path(file.getPath().toString()), true);
                }
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
        System.out.println("GZIP IS END!!!");
    }
    private static void UNDirectory(String inputPath) {
        try {
            FileStatus[] status = fs.listStatus(new Path(inputPath));
            for (FileStatus file : status) {
                System.out.println("====file"+file);
                System.out.println("====file.getPath()"+file.getPath());
                //得到输入流
                //输出文件路径
                String outputPath = file.getPath().toString().substring(0,  file.getPath().toString().length() - 3) + ".csv";
                System.out.println("====outputPath"+outputPath);
                //得到压缩输出流
                CompressionInputStream inputStream = codec.createInputStream(fs.open(new Path(file.getPath().toString())));
                IOUtils.copyBytes(inputStream, fs.create(new Path(outputPath)), conf, true);
                System.out.println("COMPRESS SUCCESS:::" + outputPath);
                if (fs.exists(new Path(outputPath))) {
                    fs.delete(new Path(file.getPath().toString()), true);
                }
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
        System.out.println("GZIP IS END!!!");
    }