使用java读取gz压缩的hdfs文件
根据hdfs文件的后缀类型自动识别并解压:
Path hdfsPath = new Path(args[0]);
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(hdfsPath.toUri(),conf);
CompressionCodecFactory factory = new CompressionCodecFactory(conf);
CompressionCodec codec = factory.getCodec(hdfsPath);
FSDataInputStream inputStream = fs.open(hdfsPath);
BufferedReader reader = null;
try {
if (codec == null) {
reader = new BufferedReader(new InputStreamReader(inputStream));
} else {
CompressionInputStream comInputStream = codec.createInputStream(inputStream);
reader = new BufferedReader(new InputStreamReader(comInputStream));
}
} catch (Exception e) {
e.printStackTrace();
}