MapReduce中的数据压缩案例实践

For Coding

于 2020-05-02 15:08:28 发布

阅读量167

点赞数

分类专栏：大数据文章标签：大数据 hadoop mapreduce

本文链接：https://blog.csdn.net/m0_46689661/article/details/105890701

版权

大数据专栏收录该内容

15 篇文章 3 订阅

订阅专栏

代码实现

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.CompressionCodecFactory;
import org.apache.hadoop.io.compress.CompressionInputStream;
import org.apache.hadoop.io.compress.CompressionOutputStream;
import org.apache.hadoop.util.ReflectionUtils;

public class CompressDemo {

    public static void main(String[] args) throws Exception {
        // testCompress("e:/input/compress/web.txt", "org.apache.hadoop.io.compress.DefaultCodec", "e:/output/web");
        testUpCompress("E:/output/web.deflate", "e:/output/web.tx");
    }

    /**
     * 压缩
     * @param inputFileName
     * @param method
     * @param outputFileName
     * @throws Exception
     */
    public static void testCompress(String inputFileName, String method, String outputFileName) throws Exception {
        Configuration conf = new Configuration();
        // 1 获取输入流
        FSDataInputStream fis = FileSystem.get(conf).open(new Path(inputFileName));
        // 2 获取输出流
        Class<?> codeClassName = Class.forName(method);
        CompressionCodec compressionCodec = (CompressionCodec) ReflectionUtils.newInstance(codeClassName, conf);
        // System.out.println(compressionCodec.getDefaultExtension());  //.bz2
        CompressionOutputStream fos =
                compressionCodec.createOutputStream(FileSystem.get(conf).create(new Path(outputFileName + compressionCodec.getDefaultExtension())));
        // 3 流对拷
        IOUtils.copyBytes(fis, fos, conf);

        // 4 关闭资源
        IOUtils.closeStream(fos);
        IOUtils.closeStream(fis);
    }

    /**
     * 解压
     * @param inputFileName
     * @param outputFileName
     * @throws Exception
     */
    public static void testUpCompress(String inputFileName, String outputFileName) throws Exception {
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(conf);
        // 1 获取输入流
        CompressionCodec codec = new CompressionCodecFactory(conf).getCodec(new Path(inputFileName));
        CompressionInputStream fis = codec.createInputStream(fs.open(new Path(inputFileName)));
        // 2 获取输出流
        FSDataOutputStream fos = fs.create(new Path(outputFileName));
        // 3 流对拷
        IOUtils.copyBytes(fis, fos, conf);

        // 4 关闭资源
        IOUtils.closeStream(fos);
        IOUtils.closeStream(fis);
    }
}