代码实现
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.CompressionCodecFactory;
import org.apache.hadoop.io.compress.CompressionInputStream;
import org.apache.hadoop.io.compress.CompressionOutputStream;
import org.apache.hadoop.util.ReflectionUtils;
public class CompressDemo {
public static void main(String[] args) throws Exception {
testUpCompress("E:/output/web.deflate", "e:/output/web.tx");
}
public static void testCompress(String inputFileName, String method, String outputFileName) throws Exception {
Configuration conf = new Configuration();
FSDataInputStream fis = FileSystem.get(conf).open(new Path(inputFileName));
Class<?> codeClassName = Class.forName(method);
CompressionCodec compressionCodec = (CompressionCodec) ReflectionUtils.newInstance(codeClassName, conf);
CompressionOutputStream fos =
compressionCodec.createOutputStream(FileSystem.get(conf).create(new Path(outputFileName + compressionCodec.getDefaultExtension())));
IOUtils.copyBytes(fis, fos, conf);
IOUtils.closeStream(fos);
IOUtils.closeStream(fis);
}
public static void testUpCompress(String inputFileName, String outputFileName) throws Exception {
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(conf);
CompressionCodec codec = new CompressionCodecFactory(conf).getCodec(new Path(inputFileName));
CompressionInputStream fis = codec.createInputStream(fs.open(new Path(inputFileName)));
FSDataOutputStream fos = fs.create(new Path(outputFileName));
IOUtils.copyBytes(fis, fos, conf);
IOUtils.closeStream(fos);
IOUtils.closeStream(fis);
}
}