hive参数设置

–动态分区
set mapreduce.job.queuename=root.risk;
set hive.exec.dynamici.partition=true;
set hive.exec.dynamic.partition.mode=nonstrict;
set hive.exec.max.dynamic.partitions.pernode=100000;
set hive.exec.max.dynamic.partitions =100000;
set hive.exec.max.created.files =1000000;

–参数设置
set mapreduce.job.queuename=root.bi;
set mapreduce.task.io.sort.mb=500;
set hive.exec.orc.default.block.size=134217728;
set mapreduce.input.fileinputformat.split.maxsize = 100000000;
set hive.auto.convert.join =true ;
set hive.exec.compress.intermediate=true;
set hive.exec.reducers.bytes.per.reducer=300000000;
-----内存设置
set mapreduce.reduce.memory.mb=4096;
set mapreduce.map.memory.mb=2048;
set mapreduce.task.io.sort.mb=800;
set mapreduce.reduce.java.opts=-Djava.net.preferIPv4Stack=true -Xmx3200m;
set mapreduce.reduce.shuffle.parallelcopies=20;
set mapreduce.job.reduce.slowstart.completedmaps=0.7;
-----输出合并小文件
set hive.merge.mapfiles = true;
set hive.merge.mapredfiles = true;
set hive.merge.size.per.task = 128000000;
set hive.merge.smallfiles.avgsize=30000000;
-----输入合并小文件
set hive.input.format=org.apache.hadoop.hive.ql.io.CombineHiveInputFormat;
set mapreduce.input.fileinputformat.split.minsize.per.node=10000000;
set mapreduce.input.fileinputformat.split.minsize.per.rack=11000000;

set hive.exec.orc.default.block.size=134217728;
set mapreduce.job.queuename=root.risk;
set hive.auto.convert.join =true ;
set hive.mapjoin.smalltable.filesize=50000000;
set hive.exec.compress.intermediate=true;
------每个reduce和map处理多少数据。在处理那种只连接不过滤不汇总的数据时，处理的数据一点不缩水会很大，灵活调小这个值。
set hive.exec.reducers.bytes.per.reducer=100000000;
set mapreduce.input.fileinputformat.split.maxsize = 120000000;
-----内存设置
set mapreduce.reduce.memory.mb=8096;
set mapreduce.map.memory.mb=8048;
set mapreduce.task.io.sort.mb=800;
set mapreduce.reduce.java.opts=-Djava.net.preferIPv4Stack=true -Xmx6200m;
set mapreduce.map.java.opts=-Djava.net.preferIPv4Stack=true -Xmx6200m;
set mapreduce.reduce.shuffle.parallelcopies=20;
set mapreduce.job.reduce.slowstart.completedmaps=0.7;
-----输出合并小文件
set hive.merge.mapfiles = true;
set hive.merge.mapredfiles = true;
set hive.merge.size.per.task = 128000000;
set hive.merge.smallfiles.avgsize=30000000;
-----输入合并小文件
set hive.input.format=org.apache.hadoop.hive.ql.io.CombineHiveInputFormat;
set mapreduce.input.fileinputformat.split.minsize.per.node=10000000;
set mapreduce.input.fileinputformat.split.minsize.per.rack=11000000;
—set hive.support.quoted.identifiers=None;
set hive.exec.dynamici.partition=true;
set hive.exec.dynamic.partition.mode=nonstrict;
set hive.exec.max.dynamic.partitions.pernode=100000;
set hive.exec.max.dynamic.partitions =100000;
set hive.exec.max.created.files =1000000;