MapReduce - 压缩

压缩位置的选择

压缩参数配置

要在Hadoop中启用压缩，可以配置如下参数：

压缩的小示例代码

package MapReduceCompress;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.CompressionCodecFactory;
import org.apache.hadoop.io.compress.CompressionInputStream;
import org.apache.hadoop.io.compress.CompressionOutputStream;
import org.apache.hadoop.util.ReflectionUtils;

import java.io.*;

public class CompressAndDecompressTest {
  public static void main(String[] args) throws Exception{
    compress("F:/Codes/JavaCodes/MapReduceLearning/testdata/Join/order.txt", "org.apache.hadoop.io.compress.BZip2Codec");
    //decompress("F:/Codes/JavaCodes/MapReduceLearning/testdata/Join/order.txt.bz2");
  }

  public static String compress(String fileName, String compressMethod) throws IOException, ClassNotFoundException {
      String ret;

      FileInputStream fis = new FileInputStream(new File(fileName));

      //利用反射来从字符串获取压缩对象
      Class codecClass = Class.forName(compressMethod);
      CompressionCodec cc = (CompressionCodec) ReflectionUtils.newInstance(codecClass,new Configuration());

      ret = fileName + cc.getDefaultExtension();
      FileOutputStream fos = new FileOutputStream(new File(ret));
      CompressionOutputStream cos = cc.createOutputStream(fos);

      IOUtils.copyBytes(fis,cos,1024 * 1024 * 5,true);

      return ret;
  }

  public static String decompress(String fileName) throws IOException, ClassNotFoundException {
      CompressionCodecFactory ccf = new CompressionCodecFactory(new Configuration());
      CompressionCodec codec = ccf.getCodec(new Path(fileName));

      if(codec == null){
          System.out.println("Can not find codec for file " + fileName);
          return null;
      }

      CompressionInputStream cis = codec.createInputStream(new FileInputStream(new File(fileName)));
      String ret = fileName + ".decoded";
      FileOutputStream fos = new FileOutputStream(ret);

      IOUtils.copyBytes(cis,fos,1024 * 1024 * 5,false);

      cis.close();
      fos.close();

      return ret;
  }
}