Hive项目优化测试

一：开启map输出阶段压缩;snappy
1．开启hive中间传输数据压缩功能
hive (default)>set hive.exec.compress.intermediate=true;
2．开启mapreduce中map输出压缩功能
hive (default)>set mapreduce.map.output.compress=true;
3．设置mapreduce中map输出数据的压缩方式
hive (default)>set mapreduce.map.output.compress.codec=org.apache.hadoop.io.compress.SnappyCodec;

2．开启hive最终输出数据压缩功能 outputformated
	hive (default)>set hive.exec.compress.output=true;
	2．开启mapreduce最终输出数据压缩
	hive (default)>set mapreduce.output.fileoutputformat.compress=true;
	3．设置mapreduce最终数据输出压缩方式
	hive (default)> set mapreduce.output.fileoutputformat.compress.codec =org.apache.hadoop.io.compress.SnappyCodec;
	4．设置mapreduce最终数据输出压缩为块压缩
	hive (default)> set mapreduce.output.fileoutputformat.compress.type=BLOCK;

二：Hive支持的存储数的格式主要有：TEXTFILE 、SEQUENCEFILE、ORC、PARQUET。
create table log_text (
track_time string,
url string,
session_id string,
referer string,
ip string,
end_user_id string,
city_id string
)
row format delimited fields terminated by ‘\t’
stored as textfile ;
create table log_orc(
track_time string,
url string,
session_id string,
referer string,
ip string,
end_user_id string,
city_id string
)
row format delimited fields terminated by ‘\t’
stored as orc ;
insert into table log_orc select * from log_text ;
create table log_parquet(
track_time string,
url string,
session_id string,
referer string,
ip string,
end_user_id string,
city_id string
)
row format delimited fields terminated by ‘\t’
stored as parquet ;
insert into table log_parquet select * from log_text ;