以前工作的时候曾经有个想法,就是将集群中存储的文件做一次遍历,把没有压缩的压缩掉!可一直没有时间实现,今天闲来无事,研究了一下,以下代码应该是核心了,也与大家一起分享一下:
从这个例子扩展,
import java.io.BufferedInputStream;
import java.net.URI;
import org.apache.hadoop.conf.
Configuration;
import org.apache.hadoop.fs.
FSDataInputStream;
import org.apache.hadoop.fs.
FSDataOutputStream;
import org.apache.hadoop.fs.
FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.compress.
CompressionCodec;
import org.apache.hadoop.io.compress.
CompressionOutputStream;
import org.apache.hadoop.util.
ReflectionUtils;
public class CompressUsingMR {
/**
* @param args
* @throws Exception
*/
public static void main(String[] args) throws Exception {
// TODO Auto-generated method stub
codecClassname = "com.hadoop.compression.lzo.
LzopCodec";
Class<?> codecClass = Class.forName(codecClassname);
Configuration conf = new Configuration();
CompressionCodec codec = (CompressionCodec)
ReflectionUtils.newInstance(
codecClass, conf);
String inputpath = "/user/hive/warehouse/info/ds=
2011-11-01/info.log";
String outputpath = inputpath + ".lzo";
FileSystem fs = FileSystem.get(conf);
Path inputPath = new Path(inputpath);
Path outputPath = new Path(outputpath); // 注意,这个路径应该在hdfs中不存在,
或者在用到的时候做逻辑处理
BufferedInputStream is = new BufferedInputStream(fs.open(
inputPath)); //注意创建FSDataInputStream的方式,用fs对象的open方法
FSDataOutputStream os = fs.create(outputPath); // 创建输出流
CompressionOutputStream out = codec.createOutputStream(os); // 创建压缩输出流
IOUtils.copyBytes(is, out, 4096, true); // true表示完成之后关闭输入、输出流
}
}