合并小文件
hive.merge.mapfile=true;
hive.merge.mapredfile=true;
hive.merge.size.pertask=32000000; //自定
hive.merge.smallfiles.avgsize=16000000; //自定
压缩文件
结果压缩
hive.exec.compress.output=true
mapred.output.compression.codec=org.apache.hadoop.io.compress.GzipCodec
mapred.output.compression.type=block
中间压缩
hive.exec.compress.intermediate=true
hive.intermediate.compression.code=org.apache.hadoop.io.compress.SnappyCodec
hive.intermediate.compression.type=block
动态分区
hive.mapred.mode=nonstrict
hive.exec.dynamic.partition.mode=nonstrict
hive.exec.max.dynamic.partitions.pernode=10000
groupby导致数据倾斜
hive.groupby.skewindata=true
自动mapjoin
hive.auto.convert.join=true;
自动本地运行
hive.exec.mode.local.auto=true
hive.exec.mode.local.auto.inputbytes.max=134217728L
hive.exec.mode.local.auto.tasks.max=2
对reduce个数限定
mapred.reduce.tasks=-1
hive.exec.reduces.bytes.per.reducer=1000000000L
hive.exec.reducers.max=999