hive压缩

hive

 

 

创建表

use gwk;
CREATE EXTERNAL TABLE xxxtable(
key String,
params String)
partitioned by (
        yyyymmdd string 
)
row format delimited fields terminated by '\t'
collection items terminated by ','
lines terminated by '\n'
STORED AS INPUTFORMAT 'com.hadoop.mapred.DeprecatedLzoTextInputFormat' 
OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
location '/user/hive/warehouse/xxx';

 

hive查询输出压缩

   set mapred.output.compression.codec=com.hadoop.compression.lzo.LzopCodec;
   set mapred.output.compress=true;
   set hive.exec.compress.output=true;

 

MR中输出压缩

conf.set("mapred.output.compression.codec", "com.hadoop.compression.lzo.LzopCodec");
		conf.set("mapred.output.compress", "true");
		conf.set("mapred.reduce.tasks", "0");