压缩位置的选择
压缩参数配置
要在Hadoop中启用压缩,可以配置如下参数:
压缩的小示例代码
package MapReduceCompress;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.CompressionCodecFactory;
import org.apache.hadoop.io.compress.CompressionInputStream;
import org.apache.hadoop.io.compress.CompressionOutputStream;
import org.apache.hadoop.util.ReflectionUtils;
import java.io.*;
public class CompressAndDecompressTest {
public static void main(String[] args) throws Exception{
compress("F:/Codes/JavaCodes/MapReduceLearning/testdata/Join/order.txt", "org.apache.hadoop.io.compress.BZip2Codec");
//decompress("F:/Codes/JavaCodes/MapReduceLearning/testdata/Join/order.txt.bz2");
}
public static String compress(String fileName, String compressMethod) throws IOException, ClassNotFoundException {
String ret;
FileInputStream fis = new FileInputStream(new File(fileName));
//利用反射来从字符串获取压缩对象
Class codecClass = Class.forName(compressMethod);
CompressionCodec cc = (CompressionCodec) ReflectionUtils.newInstance(codecClass,new Configuration());
ret = fileName + cc.getDefaultExtension();
FileOutputStream fos = new FileOutputStream(new File(ret));
CompressionOutputStream cos = cc.createOutputStream(fos);
IOUtils.copyBytes(fis,cos,1024 * 1024 * 5,true);
return ret;
}
public static String decompress(String fileName) throws IOException, ClassNotFoundException {
CompressionCodecFactory ccf = new CompressionCodecFactory(new Configuration());
CompressionCodec codec = ccf.getCodec(new Path(fileName));
if(codec == null){
System.out.println("Can not find codec for file " + fileName);
return null;
}
CompressionInputStream cis = codec.createInputStream(new FileInputStream(new File(fileName)));
String ret = fileName + ".decoded";
FileOutputStream fos = new FileOutputStream(ret);
IOUtils.copyBytes(cis,fos,1024 * 1024 * 5,false);
cis.close();
fos.close();
return ret;
}
}