spark和hive常用调优参数(具体需根据数据来确定大小)

SAPRK
SET spark.sql.sources.partitionOverwriteMode = DYNAMIC;
SET spark.hadoop.hive.exec.dynamic.partition.mod = nonstrict;
SET hive.exec.dynamic.partition.mode = nonstrict;
SET spark.default.parallelism = 1600;
SET spark.sql.shuffle.partitions = 1600;
SET spark.executor.memoryOverhead = 5G;
SET spark.debug.maxToStringFields = 10000;
SET spark.sql.debug.maxToStringFields = 10000;
SET spark.hadoop.hive.exec.max.dynamic.partitions = 11000;
SET spark.hadoop.hive.exec.max.dynamic.partitions.pernode = 1512;
SET spark.memory.offHeap.size = 4G;
SET spark.driver.memoryOverhead = 4G;
SET spark.memory.offHeap.enabled = TRUE;
SET spark.sql.legacy.parquet.int96RebaseModeInWrite = LEGACY;
SET spark.kryoserializer.buffer.max = 2000;
SET spark.storage.memoryFraction = 0.7;
SET spark.shuffle.memoryFraction = 0.7;
SET spark.sh